[RCCL] Clear HIP error state in CUCHECK and CUCHECKGOTO

 macros (#4617)
[rocm-systems] ROCm/rocm-systems#4617 (commit 7456cf4)

Author: Kapil-Shyam-Pawar

src/include/rocmwrap.h

@@ -37,6 +37,7 @@ typedef hsa_status_t (*PFN_hsa_amd_portable_export_dmabuf)(const void* ptr, size
     hipError_t err = cmd;				      \
     if( err != hipSuccess ) {				      \
       WARN("HIP failure '%s' at %s:%d", hipGetErrorString(err), __FILE__, __LINE__);		      \
+      (void)hipGetLastError(); /* clear sticky HIP error state */   \
       return ncclUnhandledCudaError;			      \
     }							      \
 } while(false)
@@ -45,6 +46,7 @@ typedef hsa_status_t (*PFN_hsa_amd_portable_export_dmabuf)(const void* ptr, size
     hipError_t err = cmd;				      \
     if( err != hipSuccess ) {				      \
       WARN("HIP failure '%s' at %s:%d", hipGetErrorString(err), __FILE__, __LINE__);		      \
+      (void)hipGetLastError(); /* clear sticky HIP error state */   \
       res = ncclUnhandledCudaError;			      \
       goto label;					      \
     }							      \

test/RcclWrapTests.cpp

@@ -14,6 +14,7 @@
 #include "common/ProcessIsolatedTestRunner.hpp"
 #include "debug.h"
 #include "graph/topo.h"
+#include "rocmwrap.h"
 
 namespace RcclUnitTesting
 {
@@ -157,6 +158,14 @@ static bool isProtoStrValid(const char* envStr)
     return false; // No match found
 }
 
+// Helper that exercises CUCHECK with a guaranteed-to-fail HIP call
+static ncclResult_t triggerCucheckFailure()
+{
+    CUCHECK(hipPointerGetAttribute(nullptr, HIP_POINTER_ATTRIBUTE_CONTEXT,
+                                   (hipDeviceptr_t)0x1));
+    return ncclSuccess;
+}
+
 // Helper function to validate algorithm string against known valid algorithms
 static bool isAlgoStrValid(const char* envStr)
 {
@@ -1306,4 +1315,48 @@ TEST(Rcclwrap, AllPxnTests)
     EXPECT_TRUE(allTestsPassed) << "One or more PXN process-isolated tests failed";
 }
 
+TEST(Rcclwrap, CucheckMacro_CheckStickyHipErrorOnFailure)
+{
+    hipError_t hipErr = hipSetDevice(0);
+    if(hipErr != hipSuccess)
+    {
+        GTEST_SKIP() << "No GPU available";
+    }
+
+    // Clear any pre-existing sticky error so we start clean
+    (void)hipGetLastError();
+
+    // Force a HIP failure through CUCHECK using an invalid device pointer (0x1)
+    ncclResult_t ret = triggerCucheckFailure();
+
+    EXPECT_EQ(ncclUnhandledCudaError, ret)
+        << "CUCHECK should return ncclUnhandledCudaError on failure";
+    EXPECT_EQ(hipSuccess, hipGetLastError())
+        << "CUCHECK must clear sticky HIP error after failure";
+}
+
+TEST(Rcclwrap, CucheckgotoMacro_CheckStickyHipErrorOnFailure)
+{
+    hipError_t hipErr = hipSetDevice(0);
+    if(hipErr != hipSuccess)
+    {
+        GTEST_SKIP() << "No GPU available";
+    }
+
+    // Clear any pre-existing sticky error so we start clean
+    (void)hipGetLastError();
+
+    // Force a HIP failure through CUCHECKGOTO using an invalid device pointer (0x1)
+    ncclResult_t ret = ncclSuccess;
+    CUCHECKGOTO(hipPointerGetAttribute(nullptr, HIP_POINTER_ATTRIBUTE_CONTEXT,
+                                       (hipDeviceptr_t)0x1),
+                ret, check_sticky);
+
+check_sticky:
+    EXPECT_EQ(ncclUnhandledCudaError, ret)
+        << "CUCHECKGOTO should set result to ncclUnhandledCudaError on failure";
+    EXPECT_EQ(hipSuccess, hipGetLastError())
+        << "CUCHECKGOTO must clear sticky HIP error after failure";
+}
+
 } // namespace RcclUnitTesting