diff --git a/src/usr/lib/python3/dist-packages/ztp/Downloader.py b/src/usr/lib/python3/dist-packages/ztp/Downloader.py index 7f3348d..08ef3d1 100644 --- a/src/usr/lib/python3/dist-packages/ztp/Downloader.py +++ b/src/usr/lib/python3/dist-packages/ztp/Downloader.py @@ -247,6 +247,10 @@ def getUrl(self, url=None, dst_file=None, incl_http_headers=None, is_secure=True else: break - os.chmod(dst_file, stat.S_IRWXU) + try: + os.chmod(dst_file, stat.S_IRWXU) + except FileNotFoundError: + return (20, None) + # Use curl result return (0, dst_file) diff --git a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py index 0da76d3..a5fee48 100644 --- a/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py +++ b/src/usr/lib/python3/dist-packages/ztp/ZTPSections.py @@ -91,7 +91,7 @@ def __buildDefaults(self, section): @param section (dict) Configuration Section input data read from JSON file. ''' - default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure'] + default_objs = ['ignore-result', 'reboot-on-success', 'reboot-on-failure', 'halt-on-failure', 'pre-ztp-plugin-download'] # Loop through objects and update them with default values for key in default_objs: _val = getField(section, key, bool, getCfg(key)) diff --git a/src/usr/lib/python3/dist-packages/ztp/defaults.py b/src/usr/lib/python3/dist-packages/ztp/defaults.py index 34464d3..710375e 100644 --- a/src/usr/lib/python3/dist-packages/ztp/defaults.py +++ b/src/usr/lib/python3/dist-packages/ztp/defaults.py @@ -48,6 +48,7 @@ "log-file" : "/var/log/ztp.log", \ "log-level" : "INFO", \ "monitor-startup-config" : True, \ + "pre-ztp-plugin-download" : True, \ "restart-ztp-interval": 300, \ "reboot-on-success" : False, \ "reboot-on-failure" : False, \ diff --git a/src/usr/lib/ztp/ztp-engine.py b/src/usr/lib/ztp/ztp-engine.py index 1d6ad89..7c229d9 100755 --- a/src/usr/lib/ztp/ztp-engine.py +++ b/src/usr/lib/ztp/ztp-engine.py @@ -432,13 +432,54 @@ def __evalZTPResult(self): # Check reboot on result flags and take action self.__rebootAction(self.objztpJson.ztpDict, delayed_reboot=True) + def __downloadPlugins(self): + '''! + Check and download plugins used by configuration sections + @return False - If failed to download one more plugins of a required configuration section + True - Successfully downloaded plugins of all configuration sections + ''' + + # Obtain a copy of the list of configuration sections + section_names = list(self.objztpJson.section_names) + abort = False + logger.debug('Verifying and downloading plugins user by configuration sections: %s' % ', '.join(section_names)) + for sec in sorted(section_names): + section = self.objztpJson.ztpDict.get(sec) + t = getTimestamp() + try: + # Retrieve individual section's progress + sec_status = section.get('status') + download_check = section.get('pre-ztp-plugin-download') + if sec_status == 'BOOT' and download_check: + logger.info('Verifying and downloading plugin used by the configuration section %s.' % (sec)) + updateActivity('Verifying and downloading plugin used by the configuration section %s' % sec) + # Get the appropriate plugin to be used for this configuration section + plugin = self.objztpJson.plugin(sec) + if plugin is None: + # Mark section status as failed + section['error'] = 'Unable to find or download requested plugin' + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + + except Exception as e: + logger.error('Exception [%s] encountered while downloading plugin for configuration section %s. Marking it as FAILED.' % (str(e), sec)) + section['error'] = 'Exception [%s] encountered while verifying the plugin' % (str(e)) + section['start-timestamp'] = t + self.objztpJson.updateStatus(section, 'FAILED') + if not section.get('ignore-result'): + abort = True + return abort + def __processConfigSections(self): '''! Process and execute individual configuration sections defined in ZTP JSON. Plugin for each configuration section is resolved and executed. Configuration section data is provided as command line argument to the plugin. Each and every section is processed before this function returns. - + @return False - If error encountered processing configuration sections and request restarting ZTP + True - If processing of configuration sections has been completed ''' # Obtain a copy of the list of configuration sections @@ -447,6 +488,9 @@ def __processConfigSections(self): # set temporary flags abort = False sort = True + if self.__downloadPlugins(): + logger.info('Halting ZTP as download of one or more plugins FAILED.') + return False logger.debug('Processing configuration sections: %s' % ', '.join(section_names)) # Loop through each sections till all of them are processed @@ -537,6 +581,7 @@ def __processConfigSections(self): # Check reboot on result flags self.__rebootAction(section) + return True def __processZTPJson(self): '''! @@ -598,32 +643,40 @@ def __processZTPJson(self): self.__loadZTPProfile("resume") # Process available configuration sections in ZTP JSON - self.__processConfigSections() - - # Determine ZTP result - self.__evalZTPResult() - - # Check restart ZTP condition - # ZTP result is failed and restart-ztp-on-failure is set or - _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ - self.objztpJson['restart-ztp-on-failure'] == True) - - # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set - _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ - self.objztpJson['restart-ztp-no-config'] == True and \ - self.objztpJson['config-fallback'] == False and - os.path.isfile(getCfg('config-db-json')) is False ) + _processing_completed = self.__processConfigSections() + # In test mode always mark processing as completed + if self.test_mode: + _processing_completed = True + + _restart_ztp_missing_config = False + _restart_ztp_on_failure = False + if _processing_completed: + # Determine ZTP result + self.__evalZTPResult() + + # Check restart ZTP condition + # ZTP result is failed and restart-ztp-on-failure is set or + _restart_ztp_on_failure = (self.objztpJson['status'] == 'FAILED' and \ + self.objztpJson['restart-ztp-on-failure'] == True) + + # ZTP completed and no startup-config is found, restart-ztp-no-config and config-fallback is not set + _restart_ztp_missing_config = ( (self.objztpJson['status'] == 'SUCCESS' or self.objztpJson['status'] == 'FAILED') and \ + self.objztpJson['restart-ztp-no-config'] == True and \ + self.objztpJson['config-fallback'] == False and + os.path.isfile(getCfg('config-db-json')) is False ) # Mark ZTP for restart - if _restart_ztp_missing_config or _restart_ztp_on_failure: + if not _processing_completed or _restart_ztp_missing_config or _restart_ztp_on_failure: os.remove(getCfg('ztp-json')) if os.path.isfile(getCfg('ztp-json-shadow')): - os.remove(getCfg('ztp-json-shadow')) + os.remove(getCfg('ztp-json-shadow')) self.objztpJson = None # Remove startup-config file to obtain a new one through ZTP if getCfg('monitor-startup-config') is True and os.path.isfile(getCfg('config-db-json')): os.remove(getCfg('config-db-json')) - if _restart_ztp_missing_config: + if not _processing_completed: + return ("restart", "Restarting ZTP due to error processing configuration sections") + elif _restart_ztp_missing_config: return ("restart", "ZTP completed but startup configuration '%s' not found" % (getCfg('config-db-json'))) elif _restart_ztp_on_failure: return ("restart", "ZTP completed with FAILED status") diff --git a/tests/test_ZTPJson.py b/tests/test_ZTPJson.py index b008de8..c38e215 100644 --- a/tests/test_ZTPJson.py +++ b/tests/test_ZTPJson.py @@ -191,6 +191,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "source": "/tmp/test_firmware_%s.json" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -199,6 +200,7 @@ def test_ztp_dynamic_url_invalid_arg_type(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -305,6 +307,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -313,6 +316,7 @@ def test_ztp_non_existent_plugin_section_name(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -459,6 +463,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "source": "file:///tmp/test_firmware.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -467,6 +472,7 @@ def test_ztp_url_reusing_plugin(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, @@ -526,6 +532,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): "destination": "/tmp/firmware_check.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -533,6 +540,7 @@ def test_ztp_url_reusing_plugin_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -682,6 +690,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -689,6 +698,7 @@ def test_ztp_url_could_not_interpreted(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -765,6 +775,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): "source": "http://localhost:2000/ztp/scripts/post_install.sh" } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -772,6 +783,7 @@ def test_ztp_return_another_invalid_url_section(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": None, @@ -813,6 +825,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -820,6 +833,7 @@ def test_ztp_dynamic_url_download(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -874,6 +888,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): } } }, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -881,6 +896,7 @@ def test_ztp_dynamic_url_download_2(self, tmpdir): }, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -927,6 +943,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "halt-on-failure": false, "ignore-result": false, "plugin": "graphservice", + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "status": "BOOT", @@ -935,6 +952,7 @@ def test_ztp_graphservice_do_not_exist(self, tmpdir): "config-fallback": false, "halt-on-failure": false, "ignore-result": false, + "pre-ztp-plugin-download": true, "reboot-on-failure": false, "reboot-on-success": false, "restart-ztp-no-config": true, diff --git a/tests/test_ztp_engine.py b/tests/test_ztp_engine.py index 7d73d9c..99a32fd 100644 --- a/tests/test_ztp_engine.py +++ b/tests/test_ztp_engine.py @@ -448,6 +448,93 @@ def test_ztp_json_ignore_fail(self): self.cfgSet('monitor-startup-config', True) self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_invalid_plugins(self): + '''! + Simple ZTP test with 3-sections, invalid plugin in the second section, ZTP Failed + ''' + content = """{ + "ztp": { + "0001-test-plugin": { + "message" : "0001-test-plugin", + "message-file" : "/etc/ztp.results" + }, + "0002-invalid-plugin": { + "pre-ztp-plugin-download" : false + }, + "0003-test-plugin": { + "pre-ztp-plugin-download" : false, + "message" : "0003-test-plugin", + "message-file" : "/etc/ztp.results" + } + } +}""" + expected_result = """0001-test-plugin +0003-test-plugin +""" + self.__init_ztp_data() + self.cfgSet('monitor-startup-config', False) + self.cfgSet('restart-ztp-no-config', False) + self.__write_file("/tmp/ztp_input.json", content) + self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json") + runCommand(COVERAGE + ZTP_ENGINE_CMD) + runCommand(COVERAGE + ZTP_CMD + ' status -v') + os.remove("/tmp/ztp_input.json") + objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4) + assert(jsonDict.get('ztp').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin') + result = self.__read_file("/etc/ztp.results") + assert(result == expected_result) + self.cfgSet('monitor-startup-config', True) + self.cfgSet('restart-ztp-no-config', True) + + def test_ztp_json_invalid_plugins_ignore_result(self): + '''! + Simple ZTP test with 3-sections, invalid plugin in the second section but result ignored, + ZTP Success + ''' + content = """{ + "ztp": { + "0001-test-plugin": { + "message" : "0001-test-plugin", + "message-file" : "/etc/ztp.results" + }, + "0002-invalid-plugin": { + "plugin" : { + "url" : "file:///no-such-plugin" + }, + "pre-ztp-plugin-download" : false, + "ignore-result" : true + }, + "0003-test-plugin": { + "pre-ztp-plugin-download" : false, + "message" : "0003-test-plugin", + "message-file" : "/etc/ztp.results" + } + } +}""" + expected_result = """0001-test-plugin +0003-test-plugin +""" + self.__init_ztp_data() + self.cfgSet('monitor-startup-config', False) + self.cfgSet('restart-ztp-no-config', False) + self.__write_file("/tmp/ztp_input.json", content) + self.__write_file(self.cfgGet("opt67-url"), "file:///tmp/ztp_input.json") + runCommand(COVERAGE + ZTP_ENGINE_CMD) + runCommand(COVERAGE + ZTP_CMD + ' status -v') + os.remove("/tmp/ztp_input.json") + objJson, jsonDict = JsonReader(self.cfgGet('ztp-json'), indent=4) + assert(jsonDict.get('ztp').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0001-test-plugin').get('status') == 'SUCCESS') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('status') == 'FAILED') + assert(jsonDict.get('ztp').get('0002-invalid-plugin').get('error') == 'Unable to find or download requested plugin') + result = self.__read_file("/etc/ztp.results") + assert(result == expected_result) + self.cfgSet('monitor-startup-config', True) + self.cfgSet('restart-ztp-no-config', True) + def test_ztp_json_ignore_ztp_success(self): '''! Simple ZTP test with 3-sections, Failure in 3 sections but ignore-result set in 2sections, ignore-result set in ztp, ZTP Success