Inspect PromptKit profiles during preflight

docs/internal/cli.md

@@ -39,10 +39,12 @@ in [Configuration Internals](configuration.md).
 Configuration validation without a selected pipeline checks structural
 configuration only. Validation with a selected pipeline also builds the
 effective catalog, resolves the pipeline, and verifies explicitly selected
-PromptKit profiles. Each explicit binding or validator profile is prepared
+PromptKit profiles. Each explicit binding or validator profile is inspected
-against the configured PromptKit source without performing generation, so an
+against the configured PromptKit source and backend registrations without
-unknown profile fails before pipeline preparation. Pipeline listing validates
+loading a prompt or performing generation, so an unknown or invalid profile
-configuration before returning normalized, sorted identifiers.
+fails before pipeline preparation. Credential availability remains an
+execution-time concern. Pipeline listing validates configuration before
+returning normalized, sorted identifiers.
 
 ## Production Composition
 

docs/internal/llm.md

@@ -47,22 +47,27 @@ states. With neither flag, profile behavior remains unchanged. Because
 production constructs one shared client, the selected state applies uniformly
 to module calls, retries, and LLM-backed validators for the whole run.
 
-An empty request profile lets the prompt select its configured default. The CLI
+An empty request profile lets the prompt select its configured default. Before a
-prepares every explicitly selected binding profile before a run begins, so a
+run begins, the CLI asks the adapter to inspect every explicitly selected
-missing explicit profile fails before stage execution. Calls record the profile
+binding profile. Inspection resolves the profile and its selected backend and
-actually selected by PromptKit. The recorder trims and deduplicates non-secret
+target without loading a prompt, reading credentials, admitting capacity, or
-profile identity, provider, model, selected backend ID, and effective reasoning
+contacting a provider, so a missing or invalid explicit profile fails before
-values for manifest use. Entries that differ in backend or reasoning remain
+stage execution while a valid `api_key_env` may remain unset. Calls record the
-distinct and deterministically ordered. Endpoint-only profiles retain an empty
+profile actually selected by PromptKit. The recorder trims and deduplicates
-backend ID, which the published JSON omits. Successful completion responses and
+non-secret profile identity, provider, model, selected backend ID, and
-recorded profile manifests identify the adapter provider as `promptkit`.
+effective reasoning values for manifest use. Entries that differ in backend or
+reasoning remain distinct and deterministically ordered. Endpoint-only profiles
+retain an empty backend ID, which the published JSON omits. Successful
+completion responses and recorded profile manifests identify the adapter
+provider as `promptkit`.
 
-The CLI's preparation-only engine and the production adapter use the same
+The CLI's profile-inspection engine and the production adapter use the same
-conversion helper to register the optional conventional `local` backend.
+profile-source construction to apply the configured profile directory or file
-Preflight therefore resolves the same backend membership as runtime without
+and register the optional conventional `local` backend. Preflight therefore
-performing generation. When the registration is absent, a profile selecting
+resolves the same ordinary profile source and backend membership as runtime
-`backend: local` fails preparation instead of falling back to a built-in or
+without performing generation. When the registration is absent, a profile
-endpoint-only target.
+selecting `backend: local` fails inspection instead of falling back to a
+built-in or endpoint-only target.
 
 Before execution, the adapter also contributes a non-secret checkpoint
 fingerprint for the effective PromptKit profile source. It combines the

internal/cli/promptkit_profiles.go

@@ -2,73 +2,34 @@ package cli
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"testing/fstest"
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
-	"gitea.maximumdirect.net/eric/promptkit"
 )
 
-const profileCheckPromptID = "notarius.profile.check"
-
-var profileCheckPromptFS = fstest.MapFS{
-	"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
-version: "1.0.0"
-default_profile: mistral-small-3
-inputs:
-  - name: transcript
-    required: true
-messages:
-  - role: user
-    content: "{{input \"transcript\"}}"
-output:
-  format: text
-  validation_mode: none
-  repair_attempts: 0
-`)},
-}
-
 func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
 	if len(profileIDs) == 0 {
 		return nil
 	}
-	engine, err := newProfileValidationEngine(cfg)
+	inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
 	if err != nil {
 		return fmt.Errorf("load PromptKit profiles: %w", err)
 	}
 	for _, profileID := range profileIDs {
-		if _, err := engine.Prepare(ctx, promptkit.RunRequest{
+		if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
-			PromptID:  profileCheckPromptID,
+			return err
-			ProfileID: profileID,
-			Inputs: map[string]promptkit.ArtifactRef{
-				"transcript": promptkit.Inline("profile check"),
-			},
-		}); err != nil {
-			if errors.Is(err, promptkit.ErrProfileNotFound) {
-				return fmt.Errorf("PromptKit profile %q is not configured", profileID)
-			}
-			return fmt.Errorf("validate PromptKit profile %q: %w", profileID, err)
 		}
 	}
 	return nil
 }
 
-func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
+func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
-	opts := []promptkit.Option{
+	return llm.PromptKitProfileSourceConfig{
-		promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
+		ProfileDir:   cfg.PromptKit.ProfileDir,
+		ProfileFile:  cfg.PromptKit.ProfileFile,
+		LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
 	}
-	if cfg.PromptKit.ProfileFile != "" {
-		opts = append(opts, promptkit.WithProfileFile(cfg.PromptKit.ProfileFile))
-	}
-	if localBackend := mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend); localBackend != nil {
-		opts = append(opts, llm.PromptKitLocalBackendOption(*localBackend))
-	}
-	return promptkit.NewEngine(promptkit.Config{
-		PromptDir:  "unused",
-		ProfileDir: cfg.PromptKit.ProfileDir,
-	}, opts...)
 }
 
 func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {

internal/cli/promptkit_profiles_test.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -11,44 +12,127 @@ import (
 	"testing"
 
 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
-	"gitea.maximumdirect.net/eric/promptkit"
 )
 
-func TestExplicitPromptKitProfileValidationUsesConfiguredLocalBackendWithoutGeneration(t *testing.T) {
+func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
 	var providerCalls atomic.Int32
 	server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
 		providerCalls.Add(1)
 	}))
 	defer server.Close()
 
-	profilePath := filepath.Join(t.TempDir(), "profiles.yml")
+	writeProfile := func(t *testing.T, name, content string) string {
-	if err := os.WriteFile(profilePath, []byte(`id: local-profile
+		t.Helper()
-backend: local
+		profilePath := filepath.Join(t.TempDir(), name+".yaml")
-model: local-model
+		if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
-`), 0o600); err != nil {
+			t.Fatal(err)
-		t.Fatal(err)
+		}
-	}
+		return profilePath
-	cfg := config.Default()
-	cfg.PromptKit.ProfileFile = profilePath
-	cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
-		Endpoint:         server.URL + "/v1",
-		ConcurrencyLimit: 2,
-	}
-	if err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}); err != nil {
-		t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
-	}
-	if providerCalls.Load() != 0 {
-		t.Fatalf("provider calls during configured profile validation = %d, want 0", providerCalls.Load())
 	}
+	localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
+	credentialProfile := `id: credential-profile
+endpoint: ` + server.URL + `/v1
+model: credential-model
+api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
+`
+	t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")
 
-	cfg.PromptKit.LocalBackend = nil
+	tests := []struct {
-	err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"})
+		name         string
-	if err == nil ||
+		profilePath  string
-		!strings.Contains(err.Error(), `validate PromptKit profile "local-profile"`) ||
+		profileID    string
-		!strings.Contains(err.Error(), promptkit.BackendLocal) {
+		profileDir   bool
-		t.Fatalf("validation without registration error = %v, want profile and local backend context", err)
+		localBackend bool
+		canceled     bool
+		wantErr      []string
+	}{
+		{
+			name:         "configured local backend",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "local-profile",
+			profileDir:   true,
+			localBackend: true,
+		},
+		{
+			name:        "missing local backend registration",
+			profilePath: writeProfile(t, "local-profile", localProfile),
+			profileID:   "local-profile",
+			wantErr:     []string{`inspect PromptKit profile "local-profile"`, `backend "local"`},
+		},
+		{
+			name:         "absent profile",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "absent-profile",
+			localBackend: true,
+			wantErr:      []string{`PromptKit profile "absent-profile" is not configured`},
+		},
+		{
+			name:        "malformed profile",
+			profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
+			profileID:   "malformed-profile",
+			wantErr:     []string{`inspect PromptKit profile "malformed-profile"`},
+		},
+		{
+			name:        "invalid profile source",
+			profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
+			profileID:   "missing-profile",
+			wantErr:     []string{"load PromptKit profiles", "failed to access source file"},
+		},
+		{
+			name:        "credential environment intentionally unset",
+			profilePath: writeProfile(t, "credential-profile", credentialProfile),
+			profileID:   "credential-profile",
+		},
+		{
+			name:         "canceled inspection",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "local-profile",
+			localBackend: true,
+			canceled:     true,
+			wantErr:      []string{"context canceled"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := config.Default()
+			if tt.profileDir {
+				cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
+			} else {
+				cfg.PromptKit.ProfileFile = tt.profilePath
+			}
+			if tt.localBackend {
+				cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
+					Endpoint:         server.URL + "/v1",
+					ConcurrencyLimit: 2,
+				}
+			}
+			ctx := context.Background()
+			if tt.canceled {
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithCancel(ctx)
+				cancel()
+			}
+			err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
+			if len(tt.wantErr) == 0 {
+				if err != nil {
+					t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
+			}
+			if tt.canceled && !errors.Is(err, context.Canceled) {
+				t.Fatalf("canceled inspection error = %v, want context canceled", err)
+			}
+			for _, want := range tt.wantErr {
+				if !strings.Contains(err.Error(), want) {
+					t.Fatalf("validation error = %q, want %q", err, want)
+				}
+			}
+		})
 	}
 	if providerCalls.Load() != 0 {
-		t.Fatalf("provider calls after missing-registration validation = %d, want 0", providerCalls.Load())
+		t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
 	}
 }

internal/framework/llm/promptkit_client.go

@@ -61,27 +61,23 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
 	if cfg.Assets == nil {
 		return nil, fmt.Errorf("PromptKit client assets must not be nil")
 	}
-	if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" {
+	profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{
-		return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
+		ProfileDir:   cfg.ProfileDir,
+		ProfileFile:  cfg.ProfileFile,
+		LocalBackend: cfg.LocalBackend,
+	})
+	if err != nil {
+		return nil, err
 	}
 	options, err := cfg.Assets.PromptKitOptions()
 	if err != nil {
 		return nil, err
 	}
-	if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" {
+	options = append(options, profileOptions...)
-		options = append(options, promptkit.WithProfileFile(profileFile))
-	}
-	var localEndpoint string
-	if cfg.LocalBackend != nil {
-		localBackend := *cfg.LocalBackend
-		localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
-		localEndpoint = localBackend.Endpoint
-		options = append(options, PromptKitLocalBackendOption(localBackend))
-	}
 	options = append(options, cfg.EngineOptions...)
 
 	engine, err := promptkit.NewEngine(promptkit.Config{
-		ProfileDir: strings.TrimSpace(cfg.ProfileDir),
+		ProfileDir: profileSource.ProfileDir,
 		Timeout:    cfg.Timeout,
 		HTTPClient: cfg.HTTPClient,
 	}, options...)
@@ -100,9 +96,9 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) {
 	return &PromptKitClient{
 		engine:          engine,
 		recorder:        recorder,
-		profileDir:      strings.TrimSpace(cfg.ProfileDir),
+		profileDir:      profileSource.ProfileDir,
-		profileFile:     strings.TrimSpace(cfg.ProfileFile),
+		profileFile:     profileSource.ProfileFile,
-		localEndpoint:   localEndpoint,
+		localEndpoint:   profileSource.localEndpoint(),
 		reasoningEffort: reasoningEffort,
 	}, nil
 }

internal/framework/llm/promptkit_profiles.go

@@ -0,0 +1,114 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/promptkit"
+)
+
+type PromptKitProfileSourceConfig struct {
+	ProfileDir   string
+	ProfileFile  string
+	LocalBackend *PromptKitLocalBackendConfig
+}
+
+func (c PromptKitProfileSourceConfig) localEndpoint() string {
+	if c.LocalBackend == nil {
+		return ""
+	}
+	return c.LocalBackend.Endpoint
+}
+
+type PromptKitProfileInspector struct {
+	engine *promptkit.Engine
+}
+
+type PromptKitProfileInspection struct {
+	ProfileID             string
+	BackendID             string
+	Model                 string
+	CredentialEnvironment string
+	CredentialRequired    bool
+}
+
+type PromptKitProfileInspectionError struct {
+	ProfileID string
+	err       error
+}
+
+func (e *PromptKitProfileInspectionError) Error() string {
+	if errors.Is(e.err, promptkit.ErrProfileNotFound) {
+		return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID)
+	}
+	return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err)
+}
+
+func (e *PromptKitProfileInspectionError) Unwrap() error {
+	return e.err
+}
+
+func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) {
+	source, options, err := promptKitProfileSourceEngineOptions(cfg)
+	if err != nil {
+		return nil, err
+	}
+	engine, err := promptkit.NewEngine(promptkit.Config{
+		PromptDir:  ".",
+		ProfileDir: source.ProfileDir,
+	}, options...)
+	if err != nil {
+		return nil, fmt.Errorf("create PromptKit profile inspector: %w", err)
+	}
+	return &PromptKitProfileInspector{engine: engine}, nil
+}
+
+func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileID string) (PromptKitProfileInspection, error) {
+	if i == nil || i.engine == nil {
+		return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil")
+	}
+	profileID = strings.TrimSpace(profileID)
+	inspection, err := i.engine.InspectProfile(ctx, profileID)
+	if err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return PromptKitProfileInspection{}, ctxErr
+		}
+		return PromptKitProfileInspection{}, &PromptKitProfileInspectionError{
+			ProfileID: profileID,
+			err:       err,
+		}
+	}
+	return PromptKitProfileInspection{
+		ProfileID:             inspection.ProfileID,
+		BackendID:             strings.TrimSpace(inspection.EffectiveModelParams.BackendID),
+		Model:                 strings.TrimSpace(inspection.EffectiveModelParams.Model),
+		CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv),
+		CredentialRequired:    inspection.APIKeyRequired,
+	}, nil
+}
+
+func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) {
+	source := PromptKitProfileSourceConfig{
+		ProfileDir:  strings.TrimSpace(cfg.ProfileDir),
+		ProfileFile: strings.TrimSpace(cfg.ProfileFile),
+	}
+	if source.ProfileDir != "" && source.ProfileFile != "" {
+		return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive")
+	}
+	if cfg.LocalBackend != nil {
+		localBackend := *cfg.LocalBackend
+		localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint)
+		source.LocalBackend = &localBackend
+	}
+
+	var options []promptkit.Option
+	if source.ProfileFile != "" {
+		options = append(options, promptkit.WithProfileFile(source.ProfileFile))
+	}
+	if source.LocalBackend != nil {
+		options = append(options, PromptKitLocalBackendOption(*source.LocalBackend))
+	}
+	return source, options, nil
+}