cephfs: pin subvolume in its own subvolume group

The "fs subvolume pin" manager command was issued through go-ceph's
PinSubVolume helper, which does not accept a subvolume group. Ceph then
looked the subvolume up in the default group and returned ENOENT for
subvolumes created in a non-default group, which is the case for
ceph-csi (and the e2e test uses the "e2e" group):

  failed to pin subvolume csi-vol-... in fs myfs: rados: ret=-2,
  No such file or directory: "subvolume 'csi-vol-...' does not exist"

Issue the "fs subvolume pin" command directly with group_name set, via
a new MgrCommand passthrough on ClusterConnection, so pinning works for
any subvolume group.

Assisted-by: goose <noreply@block.xyz>
Signed-off-by: Ismael Puerto Freire <ismaelpf@inditex.com>

Author: ismaelpuerto

internal/cephfs/core/volume.go

@@ -18,6 +18,7 @@ package core
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"path"
@@ -333,24 +334,35 @@ func checkSubvolumeHasFeature(feature string, subVolFeatures []string) bool {
 // pinType can be "export", "distributed" or "random".
 // pinSetting is the value for the pin (e.g. MDS rank "2", "true"/"false").
 //
-// NOTE: go-ceph's PinSubVolume does not accept a subvolume group argument
-// (only PinSubVolumeGroup does, and that pins the group, not an individual
-// subvolume). As a result the pin is applied against Ceph's default subvolume
-// group. A subvolume group can be configured per StorageClass in ceph-csi
-// (parameter "subvolumeGroup"), so this is a known limitation until go-ceph
-// exposes a PinSubVolume variant that accepts the group.
+// go-ceph's typed PinSubVolume helper does not accept a subvolume group, so it
+// would always target Ceph's default group and fail for subvolumes created in
+// a non-default group (as ceph-csi does). To support any group, the "fs
+// subvolume pin" manager command is issued directly with the group_name set.
 //
 // Similar To:
 //
-//	ceph fs subvolume pin <vol_name> <sub_name> <pin_type> <pin_setting>
+//	ceph fs subvolume pin <vol_name> <sub_name> <pin_type> <pin_setting> \
+//	  --group_name=<group>
 func (s *subVolumeClient) PinVolume(ctx context.Context, pinType, pinSetting string) error {
-	fsa, err := s.conn.GetFSAdmin()
+	cmd := map[string]string{
+		"prefix":      "fs subvolume pin",
+		"format":      "json",
+		"vol_name":    s.FsName,
+		"sub_name":    s.VolID,
+		"group_name":  s.SubvolumeGroup,
+		"pin_type":    pinType,
+		"pin_setting": pinSetting,
+	}
+
+	buf, err := json.Marshal(cmd)
 	if err != nil {
-		return fmt.Errorf("could not get FSAdmin to pin subvolume %s: %w", s.VolID, err)
+		return fmt.Errorf("failed to marshal pin command for subvolume %s: %w", s.VolID, err)
 	}
-	_, err = fsa.PinSubVolume(s.FsName, s.VolID, pinType, pinSetting)
+
+	_, _, err = s.conn.MgrCommand([][]byte{buf})
 	if err != nil {
-		return fmt.Errorf("failed to pin subvolume %s in fs %s: %w", s.VolID, s.FsName, err)
+		return fmt.Errorf("failed to pin subvolume %s in fs %s (group %s): %w",
+			s.VolID, s.FsName, s.SubvolumeGroup, err)
 	}
 
 	return nil

internal/util/connection.go

@@ -170,3 +170,14 @@ func (cc *ClusterConnection) GetAddrs() (string, error) {
 
 	return cc.conn.GetAddrs()
 }
+
+// MgrCommand sends a raw JSON-encoded command to the Ceph manager and returns
+// its output. It is used for manager commands that are not (yet) exposed by a
+// typed go-ceph helper.
+func (cc *ClusterConnection) MgrCommand(args [][]byte) ([]byte, string, error) {
+	if cc.conn == nil {
+		return nil, "", errors.New("cluster is not connected yet")
+	}
+
+	return cc.conn.MgrCommand(args)
+}