notarius/internal/cli/promptkit_profiles_test.go

package cli

import (
	"context"
	"errors"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"sync/atomic"
	"testing"
	"testing/fstest"

	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
)

func TestExplicitPromptKitProfileValidationInspectsProfilesWithoutGeneration(t *testing.T) {
	var providerCalls atomic.Int32
	server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
		providerCalls.Add(1)
	}))
	defer server.Close()

	writeProfile := func(t *testing.T, name, content string) string {
		t.Helper()
		profilePath := filepath.Join(t.TempDir(), name+".yaml")
		if err := os.WriteFile(profilePath, []byte(content), 0o600); err != nil {
			t.Fatal(err)
		}
		return profilePath
	}
	localProfile := "id: local-profile\nbackend: local\nmodel: local-model\n"
	credentialProfile := `id: credential-profile
endpoint: ` + server.URL + `/v1
model: credential-model
api_key_env: NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY
`
	t.Setenv("NOTARIUS_PROMPTKIT_PROFILE_INSPECTION_TEST_KEY", "")

	tests := []struct {
		name         string
		profilePath  string
		profileID    string
		profileDir   bool
		localBackend bool
		canceled     bool
		wantErr      []string
		rejectErr    []string
	}{
		{
			name:         "configured local backend",
			profilePath:  writeProfile(t, "local-profile", localProfile),
			profileID:    "local-profile",
			profileDir:   true,
			localBackend: true,
		},
		{
			name:        "missing local backend registration",
			profilePath: writeProfile(t, "local-profile", localProfile),
			profileID:   "local-profile",
			wantErr:     []string{`PromptKit profile "local-profile" is invalid or unreadable`},
		},
		{
			name:         "absent profile",
			profilePath:  writeProfile(t, "local-profile", localProfile),
			profileID:    "absent-profile",
			localBackend: true,
			wantErr:      []string{`PromptKit profile "absent-profile" is not configured`},
		},
		{
			name:        "malformed profile",
			profilePath: writeProfile(t, "malformed-profile", "id: malformed-profile\nbackend: [\n"),
			profileID:   "malformed-profile",
			wantErr:     []string{`PromptKit profile "malformed-profile" is invalid or unreadable`},
			rejectErr:   []string{"malformed-profile.yaml", "backend: ["},
		},
		{
			name:        "invalid profile source",
			profilePath: filepath.Join(t.TempDir(), "missing-profile.yaml"),
			profileID:   "missing-profile",
			wantErr:     []string{"load PromptKit profiles", "profile configuration is invalid or unreadable"},
		},
		{
			name:        "credential environment intentionally unset",
			profilePath: writeProfile(t, "credential-profile", credentialProfile),
			profileID:   "credential-profile",
		},
		{
			name:         "canceled inspection",
			profilePath:  writeProfile(t, "local-profile", localProfile),
			profileID:    "local-profile",
			localBackend: true,
			canceled:     true,
			wantErr:      []string{"context canceled"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			cfg := config.Default()
			if tt.profileDir {
				cfg.PromptKit.ProfileDir = filepath.Dir(tt.profilePath)
			} else {
				cfg.PromptKit.ProfileFile = tt.profilePath
			}
			if tt.localBackend {
				cfg.PromptKit.LocalBackend = &config.PromptKitLocalBackendConfig{
					Endpoint:         server.URL + "/v1",
					ConcurrencyLimit: 2,
				}
			}
			ctx := context.Background()
			if tt.canceled {
				var cancel context.CancelFunc
				ctx, cancel = context.WithCancel(ctx)
				cancel()
			}
			err := validateExplicitPromptKitProfiles(ctx, cfg, []string{tt.profileID}, nil)
			if len(tt.wantErr) == 0 {
				if err != nil {
					t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
				}
				return
			}
			if err == nil {
				t.Fatal("validateExplicitPromptKitProfiles() error = nil, want failure")
			}
			if tt.canceled && !errors.Is(err, context.Canceled) {
				t.Fatalf("canceled inspection error = %v, want context canceled", err)
			}
			for _, want := range tt.wantErr {
				if !strings.Contains(err.Error(), want) {
					t.Fatalf("validation error = %q, want %q", err, want)
				}
			}
			for _, rejected := range append(tt.rejectErr, tt.profilePath) {
				if rejected != "" && strings.Contains(err.Error(), rejected) {
					t.Fatalf("validation error = %q, must not expose %q", err, rejected)
				}
			}
		})
	}
	if providerCalls.Load() != 0 {
		t.Fatalf("provider calls during profile inspection = %d, want 0", providerCalls.Load())
	}
}

func TestExplicitPromptKitProfileValidationUsesFallbackAssets(t *testing.T) {
	assets := llm.NewAssetRegistry()
	if err := assets.RegisterFallbackProfileFS(fstest.MapFS{
		"profiles/fallback.yaml": {Data: []byte("id: fallback-profile\nendpoint: http://promptkit.test/v1\nmodel: fallback-model\n")},
	}, "profiles"); err != nil {
		t.Fatalf("RegisterFallbackProfileFS() error = %v, want nil", err)
	}
	if err := validateExplicitPromptKitProfiles(context.Background(), config.Default(), []string{"fallback-profile"}, assets); err != nil {
		t.Fatalf("validateExplicitPromptKitProfiles() error = %v, want nil", err)
	}
}