Add test failover support for DR placement control

Introduce non-destructive TestFailover action to verify secondary cluster
readiness without committing to failover.

- Add VRGActionTestFailover and update CRD enums/YAML
- Implement placement logic, cleanup, and action execution refactor
- Exclude test primaries from multi-primary checks
- Restore original placement decisions after test failover
- Treat TestFailover like Failover for resync and VolSync restore
- Skip LastAppDeploymentCluster updates during test failover
- Improve comments, readability, and lint compliance

Signed-off-by: Benamar Mekhissi <[email protected]>

Author: BenamarMk

api/v1alpha1/drplacementcontrol_types.go

@@ -132,7 +132,7 @@ const (
 	ProgressionDeleting                            = ProgressionStatus("Deleting")
 	ProgressionDeleted                             = ProgressionStatus("Deleted")
 	ProgressionActionPaused                        = ProgressionStatus("Paused")
-	ProgressionTestingFailover                        = ProgressionStatus("TestingFailover")
+	ProgressionTestingFailover                     = ProgressionStatus("TestingFailover")
 )
 
 // DRPlacementControlSpec defines the desired state of DRPlacementControl

api/v1alpha1/volumereplicationgroup_types.go

@@ -160,7 +160,7 @@ type MoverConfig struct {
 }
 
 // VRGAction which will be either a Failover or Relocate
-// +kubebuilder:validation:Enum=Failover;Relocate
+// +kubebuilder:validation:Enum=Failover;Relocate;TestFailover
 type VRGAction string
 
 // These are the valid values for VRGAction
@@ -172,6 +172,11 @@ const (
 	// Relocate, VRG was relocated to/from this cluster,
 	// the to/from is determined by VRG spec.ReplicationState values of Primary/Secondary respectively
 	VRGActionRelocate = VRGAction("Relocate")
+
+	// TestFailover, VRG is in a test failover state where the secondary is temporarily promoted
+	// to primary to verify readiness and data consistency without committing to the actual failover.
+	// Test failover is non-destructive and can be aborted to return to the original state.
+	VRGActionTestFailover = VRGAction("TestFailover")
 )
 
 type KubeObjectProtectionSpec struct {

config/crd/bases/ramendr.openshift.io_drplacementcontrols.yaml

@@ -79,6 +79,7 @@ spec:
                 enum:
                 - Failover
                 - Relocate
+                - TestFailover
                 type: string
               drPolicyRef:
                 description: DRPolicyRef is the reference to the DRPolicy participating

config/crd/bases/ramendr.openshift.io_protectedvolumereplicationgrouplists.yaml

@@ -115,6 +115,7 @@ spec:
                           enum:
                           - Failover
                           - Relocate
+                          - TestFailover
                           type: string
                         async:
                           description: VRGAsyncSpec has the parameters associated

config/crd/bases/ramendr.openshift.io_volumereplicationgroups.yaml

@@ -66,6 +66,7 @@ spec:
                 enum:
                 - Failover
                 - Relocate
+                - TestFailover
                 type: string
               async:
                 description: VRGAsyncSpec has the parameters associated with RegionalDR

internal/controller/drplacementcontrol.go

@@ -108,10 +108,35 @@ func (d *DRPCInstance) startProcessing() bool {
 func (d *DRPCInstance) processPlacement() (bool, error) {
 	d.log.Info("Process DRPC Placement", "DRAction", d.instance.Spec.Action)
 
+	// Handle test failover cleanup when switching away from test failover to another action
 	if d.instance.Spec.Action != rmn.ActionTestFailover {
 		rmnutil.AddAnnotation(d.instance, "ramendr.openshift.io/last-action", string(d.instance.Spec.Action))
+
+		// If we're exiting a test failover (were in TestingFailover progression), clean up placement
+		// decisions that were retained during the test, then requeue for the new action.
+		if d.instance.Status.Progression == rmn.ProgressionTestingFailover {
+			for clusterName, vrg := range d.vrgs {
+				// Find the primary VRG that was used for testing and clean up after it
+				if isVRGPrimary(vrg) && vrg.Spec.Action == rmn.VRGActionTestFailover {
+					// Remove cluster decision that was retained during test failover
+					if err := d.reconciler.removeClusterDecisionAfterTestFailover(d.ctx, d.userPlacement, clusterName); err != nil {
+						return false, err
+					}
+
+					// Reset progression state back to completed for the new action
+					d.setProgression(rmn.ProgressionCompleted)
+
+					// Requeue to process the new action with a clean slate (placement decisions reset)
+					return false, nil
+				}
+			}
+		}
 	}
 
+	return d.executeAction()
+}
+
+func (d *DRPCInstance) executeAction() (bool, error) {
 	switch d.instance.Spec.Action {
 	case rmn.ActionFailover:
 		return d.RunFailover()
@@ -150,13 +175,7 @@ func (d *DRPCInstance) RunInitialDeployment() (bool, error) {
 	// Check if we already deployed in the homeCluster or elsewhere
 	deployed, clusterName := d.isDeployed(homeCluster)
 	if deployed && clusterName != homeCluster {
-		err := d.ensureVRGManifestWork(clusterName)
-		if err != nil {
-			return !done, err
-		}
-
-		// IF deployed on cluster that is not the preferred HomeCluster, then we are done
-		return done, nil
+		return d.ensureInitialDeployActionCompleted(homeCluster)
 	}
 
 	// Ensure that initial deployment is complete
@@ -398,8 +417,7 @@ func (d *DRPCInstance) RunFailover() (bool, error) {
 		if d.instance.Spec.Action == rmn.ActionTestFailover {
 			d.setProgression(rmn.ProgressionTestingFailover)
 
-			err := d.ensurePlacement(failoverCluster)
-			if err != nil {
+			if err := d.ensurePlacement(failoverCluster); err != nil {
 				return !done, err
 			}
 
@@ -967,6 +985,14 @@ func (d *DRPCInstance) ensureFailoverActionCompleted(srcCluster string) (bool, e
 	return d.ensureActionCompleted(srcCluster)
 }
 
+func (d *DRPCInstance) ensureInitialDeployActionCompleted(srcCluster string) (bool, error) {
+	// This function was added to handle cleanup in case test failover was initiated while VRG was still
+	// in the initial deployment state. It ensures proper finalization anc cleanup if necessary.
+	d.setProgression(rmn.ProgressionCleaningUp)
+
+	return d.ensureActionCompleted(srcCluster)
+}
+
 func isDiscoveredApp(drpc *rmn.DRPlacementControl) bool {
 	return drpc.Spec.ProtectedNamespaces != nil && len(*drpc.Spec.ProtectedNamespaces) > 0
 }
@@ -1150,10 +1176,12 @@ func (d *DRPCInstance) runFinalSync(homeCluster string) (bool, error) {
 }
 
 func (d *DRPCInstance) areMultipleVRGsPrimary() bool {
+	// Count actual primaries, excluding test failover primaries which are temporary and can coexist
+	// with the original primary during testing without being considered a conflicting state.
 	numOfPrimaries := 0
 
 	for _, vrg := range d.vrgs {
-		if isVRGPrimary(vrg) {
+		if isVRGPrimary(vrg) && vrg.Spec.Action != rmn.VRGActionTestFailover {
 			numOfPrimaries++
 		}
 	}
@@ -1927,6 +1955,8 @@ func vrgAction(drpcAction rmn.DRAction) rmn.VRGAction {
 		return rmn.VRGActionFailover
 	case rmn.ActionRelocate:
 		return rmn.VRGActionRelocate
+	case rmn.ActionTestFailover:
+		return rmn.VRGActionTestFailover
 	default:
 		return ""
 	}

internal/controller/drplacementcontrol_controller.go

@@ -2004,6 +2004,86 @@ func (r *DRPlacementControlReconciler) removePlacementClusterDecisionForFailover
 	return nil
 }
 
+func (r *DRPlacementControlReconciler) removeClusterDecisionAfterTestFailover(
+	ctx context.Context,
+	placement interface{},
+	clusterName string,
+) error {
+	switch obj := placement.(type) {
+	case *plrv1.PlacementRule:
+		return r.removePlacementRuleClusterDecisionAfterTestFailover(ctx, obj, clusterName)
+	case *clrapiv1beta1.Placement:
+		return r.removePlacementClusterDecisionAfterTestFailover(ctx, obj, clusterName)
+	default:
+		return fmt.Errorf("failed to find Placement or PlacementRule")
+	}
+}
+
+func (r *DRPlacementControlReconciler) removePlacementRuleClusterDecisionAfterTestFailover(
+	_ context.Context,
+	_ *plrv1.PlacementRule,
+	_ string,
+) error {
+	// PlacementRule support for test failover cleanup is not yet implemented.
+	// PlacementRule is a legacy API; modern clusters use Placement (PlacementDecision).
+	return nil
+}
+
+// removePlacementClusterDecisionAfterTestFailover removes a cluster decision that matches the passed in clusterName
+// after test failover cleanup to restore the placement to its original state.
+func (r *DRPlacementControlReconciler) removePlacementClusterDecisionAfterTestFailover(
+	ctx context.Context,
+	placement *clrapiv1beta1.Placement,
+	clusterName string,
+) error {
+	plDecision, err := r.getPlacementDecisionFromPlacement(placement)
+	if err != nil {
+		return err
+	}
+
+	if plDecision == nil {
+		return nil
+	}
+
+	dropped := false
+	decisions := []clrapiv1beta1.ClusterDecision{}
+
+	for idx := range plDecision.Status.Decisions {
+		if plDecision.Status.Decisions[idx].ClusterName == clusterName {
+			dropped = true
+
+			continue
+		}
+
+		plDecision.Status.Decisions[idx].Reason = plDecision.Status.Decisions[idx].ClusterName
+		decisions = append(decisions, plDecision.Status.Decisions[idx])
+	}
+
+	if !dropped {
+		return nil
+	}
+
+	plDecision.Status = clrapiv1beta1.PlacementDecisionStatus{
+		Decisions: decisions,
+	}
+
+	if err := r.Status().Update(ctx, plDecision); err != nil {
+		return fmt.Errorf(
+			"failed to update placementDecision status to drop cluster decision (%s) for 'test' failover (%w)",
+			clusterName,
+			err,
+		)
+	}
+
+	r.Log.Info(
+		"Updated PlacementDecision to drop cluster decision for 'test' failover",
+		"ClusterName", clusterName,
+		"PlacementDecision", plDecision.Status.Decisions,
+	)
+
+	return nil
+}
+
 // retainClusterDecisionAsFailover retains a cluster decision in the placement with the reason as
 // PlacementDecisionReasonFailoverRetained
 func (r *DRPlacementControlReconciler) retainClusterDecisionAsFailover(

internal/controller/vrg_volrep.go

@@ -1263,7 +1263,8 @@ func (v *VRGInstance) autoResync(state volrep.ReplicationState) bool {
 		return false
 	}
 
-	if v.instance.Spec.Action != ramendrv1alpha1.VRGActionFailover {
+	if v.instance.Spec.Action != ramendrv1alpha1.VRGActionFailover &&
+		v.instance.Spec.Action != ramendrv1alpha1.VRGActionTestFailover {
 		return false
 	}
 

internal/controller/vrg_volsync.go

@@ -33,7 +33,8 @@ func (v *VRGInstance) restorePVsAndPVCsForVolSync() (int, error) {
 	numPVsRestored := 0
 
 	for _, rdSpec := range v.instance.Spec.VolSync.RDSpec {
-		failoverAction := v.instance.Spec.Action == ramendrv1alpha1.VRGActionFailover
+		failoverAction := (v.instance.Spec.Action == ramendrv1alpha1.VRGActionFailover) ||
+			(v.instance.Spec.Action == ramendrv1alpha1.VRGActionTestFailover)
 
 		var err error
 		// Source conditions are not needed and should not be added to vrg.status.ProtectedPVCs,