eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inspect PromptKit profiles during preflight

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 16:26:15 +00:00
parent 67b315099d
commit 4829f94157
6 changed files with 271 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
internal/cli/promptkit_profiles.go
View File

@@ -2,73 +2,34 @@ package cli
import (
"context"
"errors"
"fmt"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/promptkit"
)
const profileCheckPromptID = "notarius.profile.check"
var profileCheckPromptFS = fstest.MapFS{
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
version: "1.0.0"
default_profile: mistral-small-3
inputs:
- name: transcript
required: true
messages:
- role: user
content: "{{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
if len(profileIDs) == 0 {
return nil
}
engine, err := newProfileValidationEngine(cfg)
inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, promptkit.ErrProfileNotFound) {
return fmt.Errorf("PromptKit profile %q is not configured", profileID)
}
return fmt.Errorf("validate PromptKit profile %q: %w", profileID, err)
if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
return err
}
}
return nil
}
func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
opts := []promptkit.Option{
promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
return llm.PromptKitProfileSourceConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
}
if cfg.PromptKit.ProfileFile != "" {
opts = append(opts, promptkit.WithProfileFile(cfg.PromptKit.ProfileFile))
}
if localBackend := mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend); localBackend != nil {
opts = append(opts, llm.PromptKitLocalBackendOption(*localBackend))
}
return promptkit.NewEngine(promptkit.Config{
PromptDir: "unused",
ProfileDir: cfg.PromptKit.ProfileDir,
}, opts...)
}
func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {

138
internal/cli/promptkit_profiles_test.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
@@ -11,44 +12,127 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestExplicitPromptKitProfileValidationUsesConfiguredLocalBackendWithoutGeneration(t *testing.T) {
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
providerCalls.Add(1)
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`id: local-profile
backend: local
model: local-model
`), 0o600); err != nil {
t.Fatal(err)
}
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
if err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls during configured profile validation = %d, want 0", providerCalls.Load())
writeProfile := func(t *testing.T, name, content string) string {
t.Helper()
profilePath := filepath.Join(t.TempDir(), name+".yaml")
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return profilePath
}
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
credentialProfile := `id: credential-profile
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
cfg.PromptKit.LocalBackend = nil
err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"})
if err == nil ||
!strings.Contains(err.Error(), `validate PromptKit profile "local-profile"`) ||
!strings.Contains(err.Error(), promptkit.BackendLocal) {
t.Fatalf("validation without registration error = %v, want profile and local backend context", err)
tests := []struct {
name string
profilePath string
profileID string
profileDir bool
localBackend bool
canceled bool
wantErr []string
}{
{
name: "configured local backend",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
profileDir: true,
localBackend: true,
},
{
name: "missing local backend registration",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
wantErr: []string{`inspect PromptKit profile "local-profile"`, `backend "local"`},
},
{
name: "absent profile",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "absent-profile",
localBackend: true,
wantErr: []string{`PromptKit profile "absent-profile" is not configured`},
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
profileID: "malformed-profile",
wantErr: []string{`inspect PromptKit profile "malformed-profile"`},
},
{
name: "invalid profile source",
profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
profileID: "missing-profile",
wantErr: []string{"load PromptKit profiles", "failed to access source file"},
},
{
name: "credential environment intentionally unset",
profilePath: writeProfile(t, "credential-profile", credentialProfile),
profileID: "credential-profile",
},
{
name: "canceled inspection",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
localBackend: true,
canceled: true,
wantErr: []string{"context canceled"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := config.Default()
if tt.profileDir {
cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
} else {
cfg.PromptKit.ProfileFile = tt.profilePath
}
if tt.localBackend {
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
}
ctx := context.Background()
if tt.canceled {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
return
}
if err == nil {
t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
}
if tt.canceled && !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection error = %v, want context canceled", err)
}
for _, want := range tt.wantErr {
if !strings.Contains(err.Error(), want) {
t.Fatalf("validation error = %q, want %q", err, want)
}
}
})
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls after missing-registration validation = %d, want 0", providerCalls.Load())
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}

28
internal/framework/llm/promptkit_client.go
View File

@@ -61,27 +61,23 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("PromptKit client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{
ProfileDir: cfg.ProfileDir,
ProfileFile: cfg.ProfileFile,
LocalBackend: cfg.LocalBackend,
})
if err != nil {
return nil, err
}
options, err := cfg.Assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, promptkit.WithProfileFile(profileFile))
}
var localEndpoint string
if cfg.LocalBackend != nil {
localBackend := *cfg.LocalBackend
localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
localEndpoint = localBackend.Endpoint
options = append(options, PromptKitLocalBackendOption(localBackend))
}
options = append(options, profileOptions...)
options = append(options, cfg.EngineOptions...)
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
ProfileDir: profileSource.ProfileDir,
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
@@ -100,9 +96,9 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
localEndpoint: localEndpoint,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
reasoningEffort: reasoningEffort,
}, nil
}

114
internal/framework/llm/promptkit_profiles.go Normal file
View File

@@ -0,0 +1,114 @@
package llm
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit"
)
type PromptKitProfileSourceConfig struct {
ProfileDir string
ProfileFile string
LocalBackend *PromptKitLocalBackendConfig
}
func (c PromptKitProfileSourceConfig) localEndpoint() string {
if c.LocalBackend == nil {
return ""
}
return c.LocalBackend.Endpoint
}
type PromptKitProfileInspector struct {
engine *promptkit.Engine
}
type PromptKitProfileInspection struct {
ProfileID string
BackendID string
Model string
CredentialEnvironment string
CredentialRequired bool
}
type PromptKitProfileInspectionError struct {
ProfileID string
err error
}
func (e *PromptKitProfileInspectionError) Error() string {
if errors.Is(e.err, promptkit.ErrProfileNotFound) {
return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID)
}
return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err)
}
func (e *PromptKitProfileInspectionError) Unwrap() error {
return e.err
}
func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg)
if err != nil {
return nil, err
}
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: ".",
ProfileDir: source.ProfileDir,
}, options...)
if err != nil {
return nil, fmt.Errorf("create PromptKit profile inspector: %w", err)
}
return &PromptKitProfileInspector{engine: engine}, nil
}
func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileID string) (PromptKitProfileInspection, error) {
if i == nil || i.engine == nil {
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
}
profileID = strings.TrimSpace(profileID)
inspection, err := i.engine.InspectProfile(ctx, profileID)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return PromptKitProfileInspection{}, ctxErr
}
return PromptKitProfileInspection{}, &PromptKitProfileInspectionError{
ProfileID: profileID,
err: err,
}
}
return PromptKitProfileInspection{
ProfileID: inspection.ProfileID,
BackendID: strings.TrimSpace(inspection.EffectiveModelParams.BackendID),
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
CredentialRequired: inspection.APIKeyRequired,
}, nil
}
func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) {
source := PromptKitProfileSourceConfig{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
ProfileFile: strings.TrimSpace(cfg.ProfileFile),
}
if source.ProfileDir != "" && source.ProfileFile != "" {
return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
}
if cfg.LocalBackend != nil {
localBackend := *cfg.LocalBackend
localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
source.LocalBackend = &localBackend
}
var options []promptkit.Option
if source.ProfileFile != "" {
options = append(options, promptkit.WithProfileFile(source.ProfileFile))
}
if source.LocalBackend != nil {
options = append(options, PromptKitLocalBackendOption(*source.LocalBackend))
}
return source, options, nil
}