eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inspect PromptKit profiles during preflight

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-03 16:26:15 +00:00
parent 67b315099d
commit 4829f94157
6 changed files with 271 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
docs/internal/cli.md
View File

@@ -39,10 +39,12 @@ in [Configuration Internals](configuration.md).
Configuration validation without a selected pipeline checks structural
configuration only. Validation with a selected pipeline also builds the
effective catalog, resolves the pipeline, and verifies explicitly selected
PromptKit profiles. Each explicit binding or validator profile is prepared
against the configured PromptKit source without performing generation, so an
unknown profile fails before pipeline preparation. Pipeline listing validates
configuration before returning normalized, sorted identifiers.
PromptKit profiles. Each explicit binding or validator profile is inspected
against the configured PromptKit source and backend registrations without
loading a prompt or performing generation, so an unknown or invalid profile
fails before pipeline preparation. Credential availability remains an
execution-time concern. Pipeline listing validates configuration before
returning normalized, sorted identifiers.
## Production Composition

35
docs/internal/llm.md
View File

@@ -47,22 +47,27 @@ states. With neither flag, profile behavior remains unchanged. Because
production constructs one shared client, the selected state applies uniformly
to module calls, retries, and LLM-backed validators for the whole run.
An empty request profile lets the prompt select its configured default. The CLI
prepares every explicitly selected binding profile before a run begins, so a
missing explicit profile fails before stage execution. Calls record the profile
actually selected by PromptKit. The recorder trims and deduplicates non-secret
profile identity, provider, model, selected backend ID, and effective reasoning
values for manifest use. Entries that differ in backend or reasoning remain
distinct and deterministically ordered. Endpoint-only profiles retain an empty
backend ID, which the published JSON omits. Successful completion responses and
recorded profile manifests identify the adapter provider as `promptkit`.
An empty request profile lets the prompt select its configured default. Before a
run begins, the CLI asks the adapter to inspect every explicitly selected
binding profile. Inspection resolves the profile and its selected backend and
target without loading a prompt, reading credentials, admitting capacity, or
contacting a provider, so a missing or invalid explicit profile fails before
stage execution while a valid `api_key_env` may remain unset. Calls record the
profile actually selected by PromptKit. The recorder trims and deduplicates
non-secret profile identity, provider, model, selected backend ID, and
effective reasoning values for manifest use. Entries that differ in backend or
reasoning remain distinct and deterministically ordered. Endpoint-only profiles
retain an empty backend ID, which the published JSON omits. Successful
completion responses and recorded profile manifests identify the adapter
provider as `promptkit`.
The CLI's preparation-only engine and the production adapter use the same
conversion helper to register the optional conventional `local` backend.
Preflight therefore resolves the same backend membership as runtime without
performing generation. When the registration is absent, a profile selecting
`backend: local` fails preparation instead of falling back to a built-in or
endpoint-only target.
The CLI's profile-inspection engine and the production adapter use the same
profile-source construction to apply the configured profile directory or file
and register the optional conventional `local` backend. Preflight therefore
resolves the same ordinary profile source and backend membership as runtime
without performing generation. When the registration is absent, a profile
selecting `backend: local` fails inspection instead of falling back to a
built-in or endpoint-only target.
Before execution, the adapter also contributes a non-secret checkpoint
fingerprint for the effective PromptKit profile source. It combines the

55
internal/cli/promptkit_profiles.go
View File

@@ -2,73 +2,34 @@ package cli
import (
"context"
"errors"
"fmt"
"testing/fstest"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
"gitea.maximumdirect.net/eric/promptkit"
)
const profileCheckPromptID = "notarius.profile.check"
var profileCheckPromptFS = fstest.MapFS{
"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
version: "1.0.0"
default_profile: mistral-small-3
inputs:
- name: transcript
required: true
messages:
- role: user
content: "{{input \"transcript\"}}"
output:
format: text
validation_mode: none
repair_attempts: 0
`)},
}
func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
if len(profileIDs) == 0 {
return nil
}
engine, err := newProfileValidationEngine(cfg)
inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
if err != nil {
return fmt.Errorf("load PromptKit profiles: %w", err)
}
for _, profileID := range profileIDs {
if _, err := engine.Prepare(ctx, promptkit.RunRequest{
PromptID: profileCheckPromptID,
ProfileID: profileID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("profile check"),
},
}); err != nil {
if errors.Is(err, promptkit.ErrProfileNotFound) {
return fmt.Errorf("PromptKit profile %q is not configured", profileID)
}
return fmt.Errorf("validate PromptKit profile %q: %w", profileID, err)
if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
return err
}
}
return nil
}
func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
opts := []promptkit.Option{
promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
return llm.PromptKitProfileSourceConfig{
ProfileDir: cfg.PromptKit.ProfileDir,
ProfileFile: cfg.PromptKit.ProfileFile,
LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
}
if cfg.PromptKit.ProfileFile != "" {
opts = append(opts, promptkit.WithProfileFile(cfg.PromptKit.ProfileFile))
}
if localBackend := mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend); localBackend != nil {
opts = append(opts, llm.PromptKitLocalBackendOption(*localBackend))
}
return promptkit.NewEngine(promptkit.Config{
PromptDir: "unused",
ProfileDir: cfg.PromptKit.ProfileDir,
}, opts...)
}
func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {

138
internal/cli/promptkit_profiles_test.go
View File

@@ -2,6 +2,7 @@ package cli
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
@@ -11,44 +12,127 @@ import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestExplicitPromptKitProfileValidationUsesConfiguredLocalBackendWithoutGeneration(t *testing.T) {
func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
var providerCalls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
providerCalls.Add(1)
}))
defer server.Close()
profilePath := filepath.Join(t.TempDir(), "profiles.yml")
if err := os.WriteFile(profilePath, []byte(`id: local-profile
backend: local
model: local-model
`), 0o600); err != nil {
t.Fatal(err)
}
cfg := config.Default()
cfg.PromptKit.ProfileFile = profilePath
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
if err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}); err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls during configured profile validation = %d, want 0", providerCalls.Load())
writeProfile := func(t *testing.T, name, content string) string {
t.Helper()
profilePath := filepath.Join(t.TempDir(), name+".yaml")
if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
return profilePath
}
localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
credentialProfile := `id: credential-profile
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
cfg.PromptKit.LocalBackend = nil
err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"})
if err == nil ||
!strings.Contains(err.Error(), `validate PromptKit profile "local-profile"`) ||
!strings.Contains(err.Error(), promptkit.BackendLocal) {
t.Fatalf("validation without registration error = %v, want profile and local backend context", err)
tests := []struct {
name string
profilePath string
profileID string
profileDir bool
localBackend bool
canceled bool
wantErr []string
}{
{
name: "configured local backend",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
profileDir: true,
localBackend: true,
},
{
name: "missing local backend registration",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
wantErr: []string{`inspect PromptKit profile "local-profile"`, `backend "local"`},
},
{
name: "absent profile",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "absent-profile",
localBackend: true,
wantErr: []string{`PromptKit profile "absent-profile" is not configured`},
},
{
name: "malformed profile",
profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
profileID: "malformed-profile",
wantErr: []string{`inspect PromptKit profile "malformed-profile"`},
},
{
name: "invalid profile source",
profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
profileID: "missing-profile",
wantErr: []string{"load PromptKit profiles", "failed to access source file"},
},
{
name: "credential environment intentionally unset",
profilePath: writeProfile(t, "credential-profile", credentialProfile),
profileID: "credential-profile",
},
{
name: "canceled inspection",
profilePath: writeProfile(t, "local-profile", localProfile),
profileID: "local-profile",
localBackend: true,
canceled: true,
wantErr: []string{"context canceled"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := config.Default()
if tt.profileDir {
cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
} else {
cfg.PromptKit.ProfileFile = tt.profilePath
}
if tt.localBackend {
cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
Endpoint: server.URL + "/v1",
ConcurrencyLimit: 2,
}
}
ctx := context.Background()
if tt.canceled {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
cancel()
}
err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
if len(tt.wantErr) == 0 {
if err != nil {
t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
}
return
}
if err == nil {
t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
}
if tt.canceled && !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection error = %v, want context canceled", err)
}
for _, want := range tt.wantErr {
if !strings.Contains(err.Error(), want) {
t.Fatalf("validation error = %q, want %q", err, want)
}
}
})
}
if providerCalls.Load() != 0 {
t.Fatalf("provider calls after missing-registration validation = %d, want 0", providerCalls.Load())
t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
}
}

28
internal/framework/llm/promptkit_client.go
View File

@@ -61,27 +61,23 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
if cfg.Assets == nil {
return nil, fmt.Errorf("PromptKit client assets must not be nil")
}
if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{
ProfileDir: cfg.ProfileDir,
ProfileFile: cfg.ProfileFile,
LocalBackend: cfg.LocalBackend,
})
if err != nil {
return nil, err
}
options, err := cfg.Assets.PromptKitOptions()
if err != nil {
return nil, err
}
if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
options = append(options, promptkit.WithProfileFile(profileFile))
}
var localEndpoint string
if cfg.LocalBackend != nil {
localBackend := *cfg.LocalBackend
localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
localEndpoint = localBackend.Endpoint
options = append(options, PromptKitLocalBackendOption(localBackend))
}
options = append(options, profileOptions...)
options = append(options, cfg.EngineOptions...)
engine, err := promptkit.NewEngine(promptkit.Config{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
ProfileDir: profileSource.ProfileDir,
Timeout: cfg.Timeout,
HTTPClient: cfg.HTTPClient,
}, options...)
@@ -100,9 +96,9 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
return &PromptKitClient{
engine: engine,
recorder: recorder,
profileDir: strings.TrimSpace(cfg.ProfileDir),
profileFile: strings.TrimSpace(cfg.ProfileFile),
localEndpoint: localEndpoint,
profileDir: profileSource.ProfileDir,
profileFile: profileSource.ProfileFile,
localEndpoint: profileSource.localEndpoint(),
reasoningEffort: reasoningEffort,
}, nil
}

114
internal/framework/llm/promptkit_profiles.go Normal file
View File

@@ -0,0 +1,114 @@
package llm
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit"
)
type PromptKitProfileSourceConfig struct {
ProfileDir string
ProfileFile string
LocalBackend *PromptKitLocalBackendConfig
}
func (c PromptKitProfileSourceConfig) localEndpoint() string {
if c.LocalBackend == nil {
return ""
}
return c.LocalBackend.Endpoint
}
type PromptKitProfileInspector struct {
engine *promptkit.Engine
}
type PromptKitProfileInspection struct {
ProfileID string
BackendID string
Model string
CredentialEnvironment string
CredentialRequired bool
}
type PromptKitProfileInspectionError struct {
ProfileID string
err error
}
func (e *PromptKitProfileInspectionError) Error() string {
if errors.Is(e.err, promptkit.ErrProfileNotFound) {
return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID)
}
return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err)
}
func (e *PromptKitProfileInspectionError) Unwrap() error {
return e.err
}
func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
source, options, err := promptKitProfileSourceEngineOptions(cfg)
if err != nil {
return nil, err
}
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: ".",
ProfileDir: source.ProfileDir,
}, options...)
if err != nil {
return nil, fmt.Errorf("create PromptKit profile inspector: %w", err)
}
return &PromptKitProfileInspector{engine: engine}, nil
}
func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileID string) (PromptKitProfileInspection, error) {
if i == nil || i.engine == nil {
return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
}
profileID = strings.TrimSpace(profileID)
inspection, err := i.engine.InspectProfile(ctx, profileID)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return PromptKitProfileInspection{}, ctxErr
}
return PromptKitProfileInspection{}, &PromptKitProfileInspectionError{
ProfileID: profileID,
err: err,
}
}
return PromptKitProfileInspection{
ProfileID: inspection.ProfileID,
BackendID: strings.TrimSpace(inspection.EffectiveModelParams.BackendID),
Model: strings.TrimSpace(inspection.EffectiveModelParams.Model),
CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
CredentialRequired: inspection.APIKeyRequired,
}, nil
}
func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) {
source := PromptKitProfileSourceConfig{
ProfileDir: strings.TrimSpace(cfg.ProfileDir),
ProfileFile: strings.TrimSpace(cfg.ProfileFile),
}
if source.ProfileDir != "" && source.ProfileFile != "" {
return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
}
if cfg.LocalBackend != nil {
localBackend := *cfg.LocalBackend
localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
source.LocalBackend = &localBackend
}
var options []promptkit.Option
if source.ProfileFile != "" {
options = append(options, promptkit.WithProfileFile(source.ProfileFile))
}
if source.LocalBackend != nil {
options = append(options, PromptKitLocalBackendOption(*source.LocalBackend))
}
return source, options, nil
}