Make PromptKit profile handling safer and more consistent

internal/framework/llm/promptkit_client_test.go

@@ -429,6 +429,33 @@ func TestPromptKitClientCheckpointFingerprintTracksProfileSource(t *testing.T) {
 	if strings.TrimSpace(fresh[0].Value) == "" {
 		t.Fatal("built-in profile fingerprint is empty")
 	}
+
+	t.Run("directory layout", func(t *testing.T) {
+		profileDir := t.TempDir()
+		firstPath := filepath.Join(profileDir, "first-profile.yaml")
+		secondPath := filepath.Join(profileDir, "second-profile.yaml")
+		content := []byte("id: directory-profile\nendpoint: http://promptkit.test/v1\nmodel: directory-model\n")
+		if err := os.WriteFile(firstPath, content, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		first, err := promptKitProfileFingerprint(profileDir, "", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := os.Rename(firstPath, secondPath); err != nil {
+			t.Fatal(err)
+		}
+		second, err := promptKitProfileFingerprint(profileDir, "", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if first == second {
+			t.Fatalf("profile-source fingerprint = %#v after source filename changed", first)
+		}
+		if strings.Contains(first.Value, firstPath) || strings.Contains(second.Value, secondPath) {
+			t.Fatalf("profile-source fingerprint exposes source path: %#v, %#v", first, second)
+		}
+	})
 }
 
 func TestPromptKitProfileFingerprintReadErrorsDoNotExposeSourcePaths(t *testing.T) {

internal/framework/llm/promptkit_profile_fingerprint.go

@@ -27,18 +27,21 @@ func promptKitProfileFingerprint(profileDir, profileFile, fallbackProfileDigest
 
 	switch {
 	case strings.TrimSpace(profileFile) != "":
-		data, err := os.ReadFile(strings.TrimSpace(profileFile))
+		cleanProfileFile := strings.TrimSpace(profileFile)
+		data, err := os.ReadFile(cleanProfileFile)
 		if err != nil {
 			return CheckpointFingerprint{}, fmt.Errorf("read PromptKit profile file for checkpoint identity")
 		}
+		writeFingerprintPart(hasher, []byte(filepath.ToSlash(filepath.Base(cleanProfileFile))))
 		writeFingerprintPart(hasher, data)
 	case strings.TrimSpace(profileDir) != "":
-		digests, err := promptKitProfileFileDigests(strings.TrimSpace(profileDir))
+		files, err := promptKitProfileFiles(strings.TrimSpace(profileDir))
 		if err != nil {
 			return CheckpointFingerprint{}, err
 		}
-		for _, digest := range digests {
-			writeFingerprintPart(hasher, digest)
+		for _, file := range files {
+			writeFingerprintPart(hasher, []byte(file.path))
+			writeFingerprintPart(hasher, file.digest)
 		}
 	}
 
@@ -58,8 +61,13 @@ func promptKitLocalBackendFingerprint(endpoint string) CheckpointFingerprint {
 	}
 }
 
-func promptKitProfileFileDigests(root string) ([][]byte, error) {
-	var digests [][]byte
+type promptKitProfileFile struct {
+	path   string
+	digest []byte
+}
+
+func promptKitProfileFiles(root string) ([]promptKitProfileFile, error) {
+	var files []promptKitProfileFile
 	err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error {
 		if walkErr != nil {
 			return walkErr
@@ -75,17 +83,24 @@ func promptKitProfileFileDigests(root string) ([][]byte, error) {
 		if err != nil {
 			return err
 		}
+		relativePath, err := filepath.Rel(root, name)
+		if err != nil {
+			return err
+		}
 		sum := sha256.Sum256(data)
-		digests = append(digests, append([]byte(nil), sum[:]...))
+		files = append(files, promptKitProfileFile{
+			path:   filepath.ToSlash(relativePath),
+			digest: append([]byte(nil), sum[:]...),
+		})
 		return nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("read PromptKit profile directory for checkpoint identity")
 	}
-	sort.Slice(digests, func(i, j int) bool {
-		return string(digests[i]) < string(digests[j])
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].path < files[j].path
 	})
-	return digests, nil
+	return files, nil
 }
 
 func writeFingerprintPart(hasher interface{ Write([]byte) (int, error) }, value []byte) {

internal/framework/llm/promptkit_profiles.go

@@ -45,16 +45,34 @@ type PromptKitProfileInspectionError struct {
 }
 
 func (e *PromptKitProfileInspectionError) Error() string {
-	if errors.Is(e.err, promptkit.ErrProfileNotFound) {
+	switch {
+	case errors.Is(e.err, promptkit.ErrProfileNotFound):
 		return fmt.Sprintf("PromptKit profile %q is not configured", e.ProfileID)
+	case errors.Is(e.err, promptkit.ErrInvalidRequest):
+		return fmt.Sprintf("PromptKit profile ID %q is invalid", e.ProfileID)
+	case errors.Is(e.err, promptkit.ErrProfileLoad):
+		return fmt.Sprintf("PromptKit profile %q is invalid or unreadable", e.ProfileID)
+	default:
+		return fmt.Sprintf("PromptKit profile %q could not be inspected", e.ProfileID)
 	}
-	return fmt.Sprintf("inspect PromptKit profile %q: %v", e.ProfileID, e.err)
 }
 
 func (e *PromptKitProfileInspectionError) Unwrap() error {
 	return e.err
 }
 
+type promptKitProfileConfigurationError struct {
+	err error
+}
+
+func (e *promptKitProfileConfigurationError) Error() string {
+	return "PromptKit profile configuration is invalid or unreadable"
+}
+
+func (e *promptKitProfileConfigurationError) Unwrap() error {
+	return e.err
+}
+
 func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptKitProfileInspector, error) {
 	source, options, err := promptKitProfileSourceEngineOptions(cfg.Source)
 	if err != nil {
@@ -74,7 +92,7 @@ func NewPromptKitProfileInspector(cfg PromptKitProfileInspectorConfig) (*PromptK
 		ProfileDir: source.ProfileDir,
 	}, options...)
 	if err != nil {
-		return nil, fmt.Errorf("create PromptKit profile inspector: %w", err)
+		return nil, &promptKitProfileConfigurationError{err: err}
 	}
 	return &PromptKitProfileInspector{engine: engine}, nil
 }