Merge pull request #363 from PalNilsson/next

2.12.5.4

Author: PalNilsson

PILOTVERSION

@@ -1 +1 @@
-2.12.4.24
+2.12.5.4

pilot/info/extinfo.py

@@ -61,8 +61,9 @@ def load_schedconfig_data(self, pandaqueues=[], priority=[], cache_time=60):
 
         cric_url = getattr(config.Information, 'queues_url', None) or 'https://atlas-cric.cern.ch/cache/schedconfig/{pandaqueue}.json'
         cric_url = cric_url.format(pandaqueue=pandaqueues[0] if len(pandaqueues) == 1 else 'pandaqueues')
+        cvmfs_path = self.get_cvmfs_path(config.Information.queues_cvmfs, 'cric_pandaqueues.json')
 
-        sources = {'CVMFS': {'url': getattr(config.Information, 'queues_cvmfs', None) or '/cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json',
+        sources = {'CVMFS': {'url': cvmfs_path,
                              'nretry': 1,
                              'fname': os.path.join(cache_dir, 'agis_schedconf.cvmfs.json')},
                    'CRIC': {'url': cric_url,
@@ -81,6 +82,23 @@ def load_schedconfig_data(self, pandaqueues=[], priority=[], cache_time=60):
 
         return self.load_data(sources, priority, cache_time)
 
+    @staticmethod
+    def get_cvmfs_path(url, fname):
+        """
+        Return a proper path for cvmfs.
+
+        :param url: URL (string).
+        :param fname: file name for CRIC JSON (string).
+        :return: cvmfs path (string).
+        """
+
+        if url:
+            cvmfs_path = url.replace('CVMFS_PATH', os.environ.get('ATLAS_SW_BASE', '/cvmfs'))
+        else:
+            cvmfs_path = '%s/atlas.cern.ch/repo/sw/local/etc/%s' % (os.environ.get('ATLAS_SW_BASE', '/cvmfs'), fname)
+
+        return cvmfs_path
+
     @classmethod
     def load_queuedata(self, pandaqueue, priority=[], cache_time=60):
         """
@@ -113,8 +131,9 @@ def jsonparser_panda(c):
 
         cric_url = getattr(config.Information, 'queues_url', None) or 'https://atlas-cric.cern.ch/cache/schedconfig/{pandaqueue}.json'
         cric_url = cric_url.format(pandaqueue=pandaqueues[0] if len(pandaqueues) == 1 else 'pandaqueues')
+        cvmfs_path = self.get_cvmfs_path(getattr(config.Information, 'queuedata_cvmfs', None), 'cric_pandaqueues.json')
 
-        sources = {'CVMFS': {'url': getattr(config.Information, 'queuedata_cvmfs', None) or '/cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json',
+        sources = {'CVMFS': {'url': cvmfs_path,
                              'nretry': 1,
                              'fname': os.path.join(cache_dir, 'agis_schedconf.cvmfs.json')},
                    'CRIC': {'url': cric_url,
@@ -159,7 +178,8 @@ def load_storage_data(self, ddmendpoints=[], priority=[], cache_time=60):
             cache_dir = os.environ.get('PILOT_HOME', '.')
 
         # list of sources to fetch ddmconf data from
-        sources = {'CVMFS': {'url': config.Information.storages_cvmfs or '/cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_ddmendpoints.json',
+        cvmfs_path = self.get_cvmfs_path(config.Information.storages_cvmfs, 'cric_ddmendpoints.json')
+        sources = {'CVMFS': {'url': cvmfs_path,
                              'nretry': 1,
                              'fname': os.path.join(cache_dir, getattr(config.Information, 'storages_cache', None) or 'agis_ddmendpoints.json')},
                    'CRIC': {'url': (getattr(config.Information, 'storages_url', None) or 'https://atlas-cric.cern.ch/cache/ddmendpoints.json'),

pilot/util/constants.py

@@ -13,8 +13,8 @@
 # Pilot version
 RELEASE = '2'   # released number should be fixed at 2 for Pilot 2
 VERSION = '12'  # version number is '1' for first real Pilot 2 release, '0' until then, increased for bigger updates
-REVISION = '4'  # revision number should be reset to '1' for every new version release, increased for small updates
-BUILD = '24'     # build number should be reset to '1' for every new development cycle
+REVISION = '5'  # revision number should be reset to '1' for every new version release, increased for small updates
+BUILD = '4'     # build number should be reset to '1' for every new development cycle
 
 SUCCESS = 0
 FAILURE = 1

pilot/util/default.cfg

@@ -144,21 +144,21 @@ cache_dir:
 # default URL value for primary source of Queuedata (can be overwritten via --queuedata-url option)
 queuedata_url: http://pandaserver.cern.ch:25085/cache/schedconfig/{pandaqueue}.all.json
 # path to queuedata JSON provided by shared filesystem
-queuedata_cvmfs: /cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json
+queuedata_cvmfs: CVMFS_PATH/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json
 # local cache filename of the queuedata json
 queuedata_cache: queuedata.json
 
 # URL for the PanDA queues API provided by Information system
 queues_url: https://atlas-cric.cern.ch/cache/schedconfig/{pandaqueue}.json
 # path to PanDA queues JSON provided by shared filesystem
-queues_cvmfs: /cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json
+queues_cvmfs: CVMFS_PATH/atlas.cern.ch/repo/sw/local/etc/cric_pandaqueues.json
 # file name of local cache for the PanDA queues JSON
 queues_cache: cric_pandaqueues.json
 
 # URL for the DDMEndpoints/storages API provided by Information system
 storages_url: https://atlas-cric.cern.ch/cache/ddmendpoints.json
 # path to storages JSON cache provided by shared filesystem
-storages_cvmfs: /cvmfs/atlas.cern.ch/repo/sw/local/etc/cric_ddmendpoints.json
+storages_cvmfs: CVMFS_PATH/atlas.cern.ch/repo/sw/local/etc/cric_ddmendpoints.json
 # file name of local cache for the storages JSON
 storages_cache: cric_ddmendpoints.json
 

pilot/util/filehandling.py

@@ -1143,7 +1143,7 @@ def get_disk_usage(start_path='.'):
         for f in filenames:
             fp = os.path.join(dirpath, f)
             # skip if it is symbolic link
-            if not os.path.islink(fp):
+            if os.path.exists(fp) and not os.path.islink(fp):
                 total_size += os.path.getsize(fp)
 
     return total_size