Inspect PromptKit profiles during preflight

2026-08-03 16:26:15 +00:00
parent 67b315099d
commit 4829f94157
6 changed files with 271 additions and 109 deletions
--- a/internal/cli/promptkit_profiles.go
+++ b/internal/cli/promptkit_profiles.go
@@ -2,73 +2,34 @@ package cli

 import (
 	"context"
-	"errors"
 	"fmt"
-	"testing/fstest"

 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
-	"gitea.maximumdirect.net/eric/promptkit"
 )

-const profileCheckPromptID = "notarius.profile.check"
-
-var profileCheckPromptFS = fstest.MapFS{
-	"prompts/profile-check.yaml": &fstest.MapFile{Data: []byte(`id: notarius.profile.check
-version: "1.0.0"
-default_profile: mistral-small-3
-inputs:
-  - name: transcript
-    required: true
-messages:
-  - role: user
-    content: "{{input \"transcript\"}}"
-output:
-  format: text
-  validation_mode: none
-  repair_attempts: 0
-`)},
-}
-
 func validateExplicitPromptKitProfiles(ctx context.Context, cfg config.Config, profileIDs []string) error {
 	if len(profileIDs) == 0 {
 		return nil
 	}
-	engine, err := newProfileValidationEngine(cfg)
+	inspector, err := llm.NewPromptKitProfileInspector(promptKitProfileSourceConfig(cfg))
 	if err != nil {
 		return fmt.Errorf("load PromptKit profiles: %w", err)
 	}
 	for _, profileID := range profileIDs {
-		if _, err := engine.Prepare(ctx, promptkit.RunRequest{
-			PromptID:  profileCheckPromptID,
-			ProfileID: profileID,
-			Inputs: map[string]promptkit.ArtifactRef{
-				"transcript": promptkit.Inline("profile check"),
-			},
-		}); err != nil {
-			if errors.Is(err, promptkit.ErrProfileNotFound) {
-				return fmt.Errorf("PromptKit profile %q is not configured", profileID)
-			}
-			return fmt.Errorf("validate PromptKit profile %q: %w", profileID, err)
+		if _, err := inspector.InspectProfile(ctx, profileID); err != nil {
+			return err
 		}
 	}
 	return nil
 }

-func newProfileValidationEngine(cfg config.Config) (*promptkit.Engine, error) {
-	opts := []promptkit.Option{
-		promptkit.WithPromptFS(profileCheckPromptFS, "prompts"),
+func promptKitProfileSourceConfig(cfg config.Config) llm.PromptKitProfileSourceConfig {
+	return llm.PromptKitProfileSourceConfig{
+		ProfileDir:   cfg.PromptKit.ProfileDir,
+		ProfileFile:  cfg.PromptKit.ProfileFile,
+		LocalBackend: mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend),
 	}
-	if cfg.PromptKit.ProfileFile != "" {
-		opts = append(opts, promptkit.WithProfileFile(cfg.PromptKit.ProfileFile))
-	}
-	if localBackend := mapPromptKitLocalBackend(cfg.PromptKit.LocalBackend); localBackend != nil {
-		opts = append(opts, llm.PromptKitLocalBackendOption(*localBackend))
-	}
-	return promptkit.NewEngine(promptkit.Config{
-		PromptDir:  "unused",
-		ProfileDir: cfg.PromptKit.ProfileDir,
-	}, opts...)
 }

 func mapPromptKitLocalBackend(cfg *config.PromptKitLocalBackendConfig) *llm.PromptKitLocalBackendConfig {
--- a/internal/cli/promptkit_profiles_test.go
+++ b/internal/cli/promptkit_profiles_test.go
@@ -2,6 +2,7 @@ package cli

 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -11,44 +12,127 @@ import (
 	"testing"

 	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
-	"gitea.maximumdirect.net/eric/promptkit"
 )

-func TestExplicitPromptKitProfileValidationUsesConfiguredLocalBackendWithoutGeneration(t *testing.T) {
+func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
 	var providerCalls atomic.Int32
 	server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
 		providerCalls.Add(1)
 	}))
 	defer server.Close()

-	profilePath := filepath.Join(t.TempDir(), "profiles.yml")
-	if err := os.WriteFile(profilePath, []byte(`id: local-profile
-backend: local
-model: local-model
-`), 0o600); err != nil {
-		t.Fatal(err)
-	}
-	cfg := config.Default()
-	cfg.PromptKit.ProfileFile = profilePath
-	cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
-		Endpoint:         server.URL + "/v1",
-		ConcurrencyLimit: 2,
-	}
-	if err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"}); err != nil {
-		t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
-	}
-	if providerCalls.Load() != 0 {
-		t.Fatalf("provider calls during configured profile validation = %d, want 0", providerCalls.Load())
+	writeProfile := func(t *testing.T, name, content string) string {
+		t.Helper()
+		profilePath := filepath.Join(t.TempDir(), name+".yaml")
+		if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		return profilePath
 	}
+	localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
+	credentialProfile := `id: credential-profile
+endpoint: ` + server.URL + `/v1
+model: credential-model
+api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
+`
+	t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")

-	cfg.PromptKit.LocalBackend = nil
-	err := validateExplicitPromptKitProfiles(context.Background(), cfg, []string{"local-profile"})
-	if err == nil ||
-		!strings.Contains(err.Error(), `validate PromptKit profile "local-profile"`) ||
-		!strings.Contains(err.Error(), promptkit.BackendLocal) {
-		t.Fatalf("validation without registration error = %v, want profile and local backend context", err)
+	tests := []struct {
+		name         string
+		profilePath  string
+		profileID    string
+		profileDir   bool
+		localBackend bool
+		canceled     bool
+		wantErr      []string
+	}{
+		{
+			name:         "configured local backend",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "local-profile",
+			profileDir:   true,
+			localBackend: true,
+		},
+		{
+			name:        "missing local backend registration",
+			profilePath: writeProfile(t, "local-profile", localProfile),
+			profileID:   "local-profile",
+			wantErr:     []string{`inspect PromptKit profile "local-profile"`, `backend "local"`},
+		},
+		{
+			name:         "absent profile",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "absent-profile",
+			localBackend: true,
+			wantErr:      []string{`PromptKit profile "absent-profile" is not configured`},
+		},
+		{
+			name:        "malformed profile",
+			profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
+			profileID:   "malformed-profile",
+			wantErr:     []string{`inspect PromptKit profile "malformed-profile"`},
+		},
+		{
+			name:        "invalid profile source",
+			profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
+			profileID:   "missing-profile",
+			wantErr:     []string{"load PromptKit profiles", "failed to access source file"},
+		},
+		{
+			name:        "credential environment intentionally unset",
+			profilePath: writeProfile(t, "credential-profile", credentialProfile),
+			profileID:   "credential-profile",
+		},
+		{
+			name:         "canceled inspection",
+			profilePath:  writeProfile(t, "local-profile", localProfile),
+			profileID:    "local-profile",
+			localBackend: true,
+			canceled:     true,
+			wantErr:      []string{"context canceled"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := config.Default()
+			if tt.profileDir {
+				cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
+			} else {
+				cfg.PromptKit.ProfileFile = tt.profilePath
+			}
+			if tt.localBackend {
+				cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
+					Endpoint:         server.URL + "/v1",
+					ConcurrencyLimit: 2,
+				}
+			}
+			ctx := context.Background()
+			if tt.canceled {
+				var cancel context.CancelFunc
+				ctx, cancel = context.WithCancel(ctx)
+				cancel()
+			}
+			err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID})
+			if len(tt.wantErr) == 0 {
+				if err != nil {
+					t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
+				}
+				return
+			}
+			if err == nil {
+				t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
+			}
+			if tt.canceled && !errors.Is(err, context.Canceled) {
+				t.Fatalf("canceled inspection error = %v, want context canceled", err)
+			}
+			for _, want := range tt.wantErr {
+				if !strings.Contains(err.Error(), want) {
+					t.Fatalf("validation error = %q, want %q", err, want)
+				}
+			}
+		})
 	}
 	if providerCalls.Load() != 0 {
-		t.Fatalf("provider calls after missing-registration validation = %d, want 0", providerCalls.Load())
+		t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
 	}
 }