diff --git a/docs/internal/cli.md b/docs/internal/cli.md index e88850e..d68f064 100644 --- a/docs/internal/cli.md +++ b/docs/internal/cli.md @@ -39,10 +39,12 @@ in [Configuration Internals](configuration.md). Configuration validation without a selected pipeline checks structural configuration only. Validation with a selected pipeline also builds the effective catalog, resolves the pipeline, and verifies explicitly selected -PromptKit profiles. Each explicit binding or validator profile is prepared -against the configured PromptKit source without performing generation, so an -unknown profile fails before pipeline preparation. Pipeline listing validates -configuration before returning normalized, sorted identifiers. +PromptKit profiles. Each explicit binding or validator profile is inspected +against the configured PromptKit source and backend registrations without +loading a prompt or performing generation, so an unknown or invalid profile +fails before pipeline preparation. Credential availability remains an +execution-time concern. Pipeline listing validates configuration before +returning normalized, sorted identifiers. ## Production Composition diff --git a/docs/internal/llm.md b/docs/internal/llm.md index c21da28..08a0903 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -47,22 +47,27 @@ states. With neither flag, profile behavior remains unchanged. Because production constructs one shared client, the selected state applies uniformly to module calls, retries, and LLM-backed validators for the whole run. -An empty request profile lets the prompt select its configured default. The CLI -prepares every explicitly selected binding profile before a run begins, so a -missing explicit profile fails before stage execution. Calls record the profile -actually selected by PromptKit. The recorder trims and deduplicates non-secret -profile identity, provider, model, selected backend ID, and effective reasoning -values for manifest use. Entries that differ in backend or reasoning remain -distinct and deterministically ordered. Endpoint-only profiles retain an empty -backend ID, which the published JSON omits. Successful completion responses and -recorded profile manifests identify the adapter provider as `promptkit`. +An empty request profile lets the prompt select its configured default. Before a +run begins, the CLI asks the adapter to inspect every explicitly selected +binding profile. Inspection resolves the profile and its selected backend and +target without loading a prompt, reading credentials, admitting capacity, or +contacting a provider, so a missing or invalid explicit profile fails before +stage execution while a valid `api_key_env` may remain unset. Calls record the +profile actually selected by PromptKit. The recorder trims and deduplicates +non-secret profile identity, provider, model, selected backend ID, and +effective reasoning values for manifest use. Entries that differ in backend or +reasoning remain distinct and deterministically ordered. Endpoint-only profiles +retain an empty backend ID, which the published JSON omits. Successful +completion responses and recorded profile manifests identify the adapter +provider as `promptkit`. -The CLI's preparation-only engine and the production adapter use the same -conversion helper to register the optional conventional `local` backend. -Preflight therefore resolves the same backend membership as runtime without -performing generation. When the registration is absent, a profile selecting -`backend: local` fails preparation instead of falling back to a built-in or -endpoint-only target. +The CLI's profile-inspection engine and the production adapter use the same +profile-source construction to apply the configured profile directory or file +and register the optional conventional `local` backend. Preflight therefore +resolves the same ordinary profile source and backend membership as runtime +without performing generation. When the registration is absent, a profile +selecting `backend: local` fails inspection instead of falling back to a +built-in or endpoint-only target. Before execution, the adapter also contributes a non-secret checkpoint fingerprint for the effective PromptKit profile source. It combines the diff --git a/internal/cli/promptkit_profiles.go b/internal/cli/promptkit_profiles.go index 7dbcabe..2933eb2 100644 --- a/internal/cli/promptkit_profiles.go +++ b/internal/cli/promptkit_profiles.go @@ -2,73 +2,34 @@ package cli import ( "context" - "errors" "fmt" - "testing/fstest" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" - "gitea.maximumdirect.net/eric/promptkit" ) -const profileCheckPromptID = "notarius.profile.check" - -var profileCheckPromptFS = fstest.MapFS{ - "prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check -version: "1.0.0" -default_profile: mistral-small-3 -inputs: - - name: transcript - required: true -messages: - - role: user - content: "{{input \"transcript\"}}" -output: - format: text - validation_mode: none - repair_attempts: 0 -`)}, -} - func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error { if len(profileIDs) == 0 { return nil } - engine, err := newProfileValidationEngine(cfg) + inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg)) if err != nil { return fmt.Errorf("load PromptKit profiles: %w", err) } for _, profileID := range profileIDs { - if _, err := engine.Prepare(ctx, promptkit.RunRequest{ - PromptID: profileCheckPromptID, - ProfileID: profileID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("profile check"), - }, - }); err != nil { - if errors.Is(err, promptkit.ErrProfileNotFound) { - return fmt.Errorf("PromptKit profile %q is not configured", profileID) - } - return fmt.Errorf("validate PromptKit profile %q: %w", profileID, err) + if _, err := inspector.InspectProfile(ctx, profileID); err != nil { + return err } } return nil } -func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) { - opts := []promptkit.Option{ - promptkit.WithPromptFS(profileCheckPromptFS, "prompts"), +func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig { + return llm.PromptKitProfileSourceConfig{ + ProfileDir: cfg.PromptKit.ProfileDir, + ProfileFile: cfg.PromptKit.ProfileFile, + LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend), } - if cfg.PromptKit.ProfileFile != "" { - opts = append(opts, promptkit.WithProfileFile(cfg.PromptKit.ProfileFile)) - } - if localBackend := mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend); localBackend != nil { - opts = append(opts, llm.PromptKitLocalBackendOption(*localBackend)) - } - return promptkit.NewEngine(promptkit.Config{ - PromptDir: "unused", - ProfileDir: cfg.PromptKit.ProfileDir, - }, opts...) } func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig { diff --git a/internal/cli/promptkit_profiles_test.go b/internal/cli/promptkit_profiles_test.go index 9b173ac..cb12f55 100644 --- a/internal/cli/promptkit_profiles_test.go +++ b/internal/cli/promptkit_profiles_test.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "net/http" "net/http/httptest" "os" @@ -11,44 +12,127 @@ import ( "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/config" - "gitea.maximumdirect.net/eric/promptkit" ) -func TestExplicitPromptKitProfileValidationUsesConfiguredLocalBackendWithoutGeneration(t *testing.T) { +func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) { var providerCalls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { providerCalls.Add(1) })) defer server.Close() - profilePath := filepath.Join(t.TempDir(), "profiles.yml") - if err := os.WriteFile(profilePath, []byte(`id: local-profile -backend: local -model: local-model -`), 0o600); err != nil { - t.Fatal(err) - } - cfg := config.Default() - cfg.PromptKit.ProfileFile = profilePath - cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{ - Endpoint: server.URL + "/v1", - ConcurrencyLimit: 2, - } - if err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}); err != nil { - t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) - } - if providerCalls.Load() != 0 { - t.Fatalf("provider calls during configured profile validation = %d, want 0", providerCalls.Load()) + writeProfile := func(t *testing.T, name, content string) string { + t.Helper() + profilePath := filepath.Join(t.TempDir(), name+".yaml") + if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return profilePath } + localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n" + credentialProfile := `id: credential-profile +endpoint: ` + server.URL + `/v1 +model: credential-model +api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY +` + t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "") - cfg.PromptKit.LocalBackend = nil - err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}) - if err == nil || - !strings.Contains(err.Error(), `validate PromptKit profile "local-profile"`) || - !strings.Contains(err.Error(), promptkit.BackendLocal) { - t.Fatalf("validation without registration error = %v, want profile and local backend context", err) + tests := []struct { + name string + profilePath string + profileID string + profileDir bool + localBackend bool + canceled bool + wantErr []string + }{ + { + name: "configured local backend", + profilePath: writeProfile(t, "local-profile", localProfile), + profileID: "local-profile", + profileDir: true, + localBackend: true, + }, + { + name: "missing local backend registration", + profilePath: writeProfile(t, "local-profile", localProfile), + profileID: "local-profile", + wantErr: []string{`inspect PromptKit profile "local-profile"`, `backend "local"`}, + }, + { + name: "absent profile", + profilePath: writeProfile(t, "local-profile", localProfile), + profileID: "absent-profile", + localBackend: true, + wantErr: []string{`PromptKit profile "absent-profile" is not configured`}, + }, + { + name: "malformed profile", + profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"), + profileID: "malformed-profile", + wantErr: []string{`inspect PromptKit profile "malformed-profile"`}, + }, + { + name: "invalid profile source", + profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"), + profileID: "missing-profile", + wantErr: []string{"load PromptKit profiles", "failed to access source file"}, + }, + { + name: "credential environment intentionally unset", + profilePath: writeProfile(t, "credential-profile", credentialProfile), + profileID: "credential-profile", + }, + { + name: "canceled inspection", + profilePath: writeProfile(t, "local-profile", localProfile), + profileID: "local-profile", + localBackend: true, + canceled: true, + wantErr: []string{"context canceled"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := config.Default() + if tt.profileDir { + cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath) + } else { + cfg.PromptKit.ProfileFile = tt.profilePath + } + if tt.localBackend { + cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{ + Endpoint: server.URL + "/v1", + ConcurrencyLimit: 2, + } + } + ctx := context.Background() + if tt.canceled { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}) + if len(tt.wantErr) == 0 { + if err != nil { + t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err) + } + return + } + if err == nil { + t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure") + } + if tt.canceled && !errors.Is(err, context.Canceled) { + t.Fatalf("canceled inspection error = %v, want context canceled", err) + } + for _, want := range tt.wantErr { + if !strings.Contains(err.Error(), want) { + t.Fatalf("validation error = %q, want %q", err, want) + } + } + }) } if providerCalls.Load() != 0 { - t.Fatalf("provider calls after missing-registration validation = %d, want 0", providerCalls.Load()) + t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load()) } } diff --git a/internal/framework/llm/promptkit_client.go b/internal/framework/llm/promptkit_client.go index 7269f11..33119d3 100644 --- a/internal/framework/llm/promptkit_client.go +++ b/internal/framework/llm/promptkit_client.go @@ -61,27 +61,23 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) { if cfg.Assets == nil { return nil, fmt.Errorf("PromptKit client assets must not be nil") } - if strings.TrimSpace(cfg.ProfileDir) != "" && strings.TrimSpace(cfg.ProfileFile) != "" { - return nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive") + profileSource, profileOptions, err := promptKitProfileSourceEngineOptions(PromptKitProfileSourceConfig{ + ProfileDir: cfg.ProfileDir, + ProfileFile: cfg.ProfileFile, + LocalBackend: cfg.LocalBackend, + }) + if err != nil { + return nil, err } options, err := cfg.Assets.PromptKitOptions() if err != nil { return nil, err } - if profileFile := strings.TrimSpace(cfg.ProfileFile); profileFile != "" { - options = append(options, promptkit.WithProfileFile(profileFile)) - } - var localEndpoint string - if cfg.LocalBackend != nil { - localBackend := *cfg.LocalBackend - localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint) - localEndpoint = localBackend.Endpoint - options = append(options, PromptKitLocalBackendOption(localBackend)) - } + options = append(options, profileOptions...) options = append(options, cfg.EngineOptions...) engine, err := promptkit.NewEngine(promptkit.Config{ - ProfileDir: strings.TrimSpace(cfg.ProfileDir), + ProfileDir: profileSource.ProfileDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, }, options...) @@ -100,9 +96,9 @@ func NewPromptKitClient(cfg PromptKitClientConfig) (*PromptKitClient, error) { return &PromptKitClient{ engine: engine, recorder: recorder, - profileDir: strings.TrimSpace(cfg.ProfileDir), - profileFile: strings.TrimSpace(cfg.ProfileFile), - localEndpoint: localEndpoint, + profileDir: profileSource.ProfileDir, + profileFile: profileSource.ProfileFile, + localEndpoint: profileSource.localEndpoint(), reasoningEffort: reasoningEffort, }, nil } diff --git a/internal/framework/llm/promptkit_profiles.go b/internal/framework/llm/promptkit_profiles.go new file mode 100644 index 0000000..9517022 --- /dev/null +++ b/internal/framework/llm/promptkit_profiles.go @@ -0,0 +1,114 @@ +package llm + +import ( + "context" + "errors" + "fmt" + "strings" + + "gitea.maximumdirect.net/eric/promptkit" +) + +type PromptKitProfileSourceConfig struct { + ProfileDir string + ProfileFile string + LocalBackend *PromptKitLocalBackendConfig +} + +func (c PromptKitProfileSourceConfig) localEndpoint() string { + if c.LocalBackend == nil { + return "" + } + return c.LocalBackend.Endpoint +} + +type PromptKitProfileInspector struct { + engine *promptkit.Engine +} + +type PromptKitProfileInspection struct { + ProfileID string + BackendID string + Model string + CredentialEnvironment string + CredentialRequired bool +} + +type PromptKitProfileInspectionError struct { + ProfileID string + err error +} + +func (e *PromptKitProfileInspectionError) Error() string { + if errors.Is(e.err, promptkit.ErrProfileNotFound) { + return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID) + } + return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err) +} + +func (e *PromptKitProfileInspectionError) Unwrap() error { + return e.err +} + +func NewPromptKitProfileInspector(cfg PromptKitProfileSourceConfig) (*PromptKitProfileInspector, error) { + source, options, err := promptKitProfileSourceEngineOptions(cfg) + if err != nil { + return nil, err + } + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: ".", + ProfileDir: source.ProfileDir, + }, options...) + if err != nil { + return nil, fmt.Errorf("create PromptKit profile inspector: %w", err) + } + return &PromptKitProfileInspector{engine: engine}, nil +} + +func (i *PromptKitProfileInspector) InspectProfile(ctx context.Context, profileID string) (PromptKitProfileInspection, error) { + if i == nil || i.engine == nil { + return PromptKitProfileInspection{}, fmt.Errorf("PromptKit profile inspector must not be nil") + } + profileID = strings.TrimSpace(profileID) + inspection, err := i.engine.InspectProfile(ctx, profileID) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return PromptKitProfileInspection{}, ctxErr + } + return PromptKitProfileInspection{}, &PromptKitProfileInspectionError{ + ProfileID: profileID, + err: err, + } + } + return PromptKitProfileInspection{ + ProfileID: inspection.ProfileID, + BackendID: strings.TrimSpace(inspection.EffectiveModelParams.BackendID), + Model: strings.TrimSpace(inspection.EffectiveModelParams.Model), + CredentialEnvironment: strings.TrimSpace(inspection.EffectiveModelParams.APIKeyEnv), + CredentialRequired: inspection.APIKeyRequired, + }, nil +} + +func promptKitProfileSourceEngineOptions(cfg PromptKitProfileSourceConfig) (PromptKitProfileSourceConfig, []promptkit.Option, error) { + source := PromptKitProfileSourceConfig{ + ProfileDir: strings.TrimSpace(cfg.ProfileDir), + ProfileFile: strings.TrimSpace(cfg.ProfileFile), + } + if source.ProfileDir != "" && source.ProfileFile != "" { + return PromptKitProfileSourceConfig{}, nil, fmt.Errorf("PromptKit profile_dir and profile_file are mutually exclusive") + } + if cfg.LocalBackend != nil { + localBackend := *cfg.LocalBackend + localBackend.Endpoint = strings.TrimSpace(localBackend.Endpoint) + source.LocalBackend = &localBackend + } + + var options []promptkit.Option + if source.ProfileFile != "" { + options = append(options, promptkit.WithProfileFile(source.ProfileFile)) + } + if source.LocalBackend != nil { + options = append(options, PromptKitLocalBackendOption(*source.LocalBackend)) + } + return source, options, nil +}