Merge branch 'dev_4.1.0' into 'master'

Dev 4.1.0
e/ob-deploy!804

Author: gys-git

_cmd.py

@@ -24,6 +24,7 @@
 import glob
 import datetime
 import copy
+import getpass
 from uuid import uuid1 as uuid, UUID
 from optparse import OptionParser, BadOptionError, Option, IndentedHelpFormatter
 
@@ -906,6 +907,10 @@ def _do_command(self, obd):
         # white_ip_list = self.get_white_ip_list()
         url = '/#/updateWelcome' if self.cmds and self.cmds[0] in ('upgrade', 'update') else ''
 
+        if self.opts.port < 1024:
+            ROOT_IO.error('The port number must be in the range of 1025-65535.')
+            return False
+
         ROOT_IO.print('start OBD WEB in 0.0.0.0:%s' % self.opts.port)
         ROOT_IO.print('please open http://{0}:{1}{2}'.format(NetUtil.get_host_ip(), self.opts.port, url))
         try:
@@ -930,7 +935,7 @@ def __init__(self):
         self.parser.add_option('-U', '--unuselibrepo', '--ulp', action='store_true', help="Disable OBD from installing the libs mirror automatically.")
         self.parser.add_option('-A', '--auto-create-tenant', '--act', action='store_true', help="Automatically create a tenant named `test` by using all the available resource of the cluster.")
         self.parser.add_option('--force-delete', action='store_true', help="Force delete, delete the registered cluster.")
-        self.parser.add_option('-s', '--strict-check', action='store_true', help="Throw errors instead of warnings when check fails.")
+        self.parser.add_option('-S', '--strict-check', action='store_true', help="Throw errors instead of warnings when check fails.")
 
     def _do_command(self, obd):
         if self.cmds:
@@ -1047,7 +1052,8 @@ def __init__(self):
         self.parser.add_option('-c', '--components', type='string', help="List of components to be started. Multiple components are separated with commas.")
         self.parser.add_option('-f', '--force-delete', action='store_true', help="Force delete, delete the registered cluster.")
         self.parser.add_option('-S', '--strict-check', action='store_true', help="Throw errors instead of warnings when check fails.")
-        self.parser.add_option('--without-parameter', '--wop', action='store_true', help='Start without parameters.')
+        self.parser.add_option('--with-parameter', '--wp', action='store_true', help='Start with parameters.')
+        self.parser.add_option('--service-names', '--sn', type='string', help='List of services to be started for PowerRAG. Multiple services are separated with commas.')
 
     def _do_command(self, obd):
         if self.cmds:
@@ -1065,6 +1071,7 @@ def __init__(self):
         super(ClusterStopCommand, self).__init__('stop', 'Stop a started cluster.')
         self.parser.add_option('-s', '--servers', type='string', help="List of servers to be stoped. Multiple servers are separated with commas.")
         self.parser.add_option('-c', '--components', type='string', help="List of components to be stoped. Multiple components are separated with commas.")
+        self.parser.add_option('--service-names', '--sn', type='string', help='List of services to be stoped for PowerRAG. Multiple services are separated with commas.')
 
     def _do_command(self, obd):
         if self.cmds:
@@ -1128,8 +1135,8 @@ def __init__(self):
 
     def _do_command(self, obd):
         if self.cmds:
-            if not getattr(self.opts, 'with_parameter', False):
-                setattr(self.opts, 'without_parameter', True)
+            # if not getattr(self.opts, 'with_parameter', False):
+            #     setattr(self.opts, 'without_parameter', True)
             obd.set_options(self.opts)
             res = obd.restart_cluster(self.cmds[0])
             self.background_telemetry_task(obd)
@@ -1227,6 +1234,8 @@ def __init__(self):
         self.parser.add_option('-e', '--executer-path', type='string', help="Executer path.", default=os.path.join(ObdCommand.OBD_INSTALL_PATH, 'lib/executer'))
         self.parser.add_option('-t', '--script-query-timeout', type='string', help="The timeout(s) for executing sql in upgrade scripts. Supported since version 4.1.0", default='')
         self.parser.add_option('--ignore-standby', '--igs', action='store_true', help="Force upgrade, before upgrade standby tenant`s cluster.")
+        self.parser.add_option('--oms-backup-path', type='string', help="Upgrade the OMS backup meta data directory", default=os.path.join(os.path.expanduser('~'), 'oms', 'meta_backup_data'))
+        self.parser.add_option('--disable-oms-backup', '--dob', action='store_true', help="disable OMS backup meta data.")
 
     def _do_command(self, obd):
         oms_upgrade_mode = None
@@ -1324,6 +1333,8 @@ def __init__(self):
         super(ClusterTenantSwitchoverCommand, self).__init__('switchover', 'Switchover primary-standby tenant.')
         self.parser.add_option('-p', '--tenant-root-password', type='string', help="tenant root password")
         self.parser.add_option('--standbyro-password', type='string', help="standbyro user password.")
+        self.parser.add_option('--standby_archive_log_uri', type='string', help='The storage directory path for standby tenant archive logs.')
+        self.parser.add_option('--primary_archive_log_uri', type='string', help='The storage directory path for primary tenant archive logs.')
 
     def init(self, cmd, args):
         super(ClusterTenantSwitchoverCommand, self).init(cmd, args)
@@ -1485,21 +1496,21 @@ class ClusterTenantRestoreCommand(ClusterMirrorCommand):
     def __init__(self):
         super(ClusterTenantRestoreCommand, self).__init__('restore', 'Restore tenant from backup.')
         self.parser.add_option('-z', '--zone', type='string', help='The zones of the tenant. example: zone1,zone2,zone3')
-        self.parser.add_option('--unit_num', type='int', help='The number of units in each zone. Default: 1.', default=1)
-        self.parser.add_option('--replica_type ', type='string', help='The replica type of the tenant.')
-        self.parser.add_option('-p', '--primary_zone', type='string', help="The primary zone of the tenant to be restored.")
+        self.parser.add_option('--unit-num', type='int', help='The number of units in each zone. Default: 1.', default=1)
+        self.parser.add_option('--replica-type ', type='string', help='The replica type of the tenant.')
+        self.parser.add_option('-p', '--primary-zone', type='string', help="The primary zone of the tenant to be restored.")
         self.parser.add_option('-T', '--timestamp', type='string', help='The timestamp to restore to.')
         self.parser.add_option('-S', '--scn', type='int', help="The SCN to restore to. Default: 0.")
-        self.parser.add_option('-s', '--ha_high_thread_score', type='int', help='The high thread score for HA. Range: [0, 100]')
+        self.parser.add_option('-s', '--ha-high-thread-score', type='int', help='The high thread score for HA. Range: [0, 100]')
         self.parser.add_option('-c', '--concurrency', type='int', help='The number of threads to use for the restore operation.')
         self.parser.add_option('-D', '--decryption', type='string', help='The decryption password for all backups. example: key1,key2,key3')
-        self.parser.add_option('-k', '--kms_encrypt_info', type='string', help='The KMS encryption information.')
-        self.parser.add_option('--memory_size', type='string', help='The memory size of the resource unit config')
-        self.parser.add_option('--max_cpu', type='int', help='The max cpu of the resource unit config, should be greater than 1.')
-        self.parser.add_option('--min_cpu', type='int', help='The min cpu of the resource unit config, should be greater than 1.If not set, the min cpu will be set to the max cpu.')
-        self.parser.add_option('--max_iops', type='int', help='The max iops of the resource unit config.If not set, the max iops will be set default value by observer.')
-        self.parser.add_option('--min_iops', type='int', help='The min iops of the resource unit config.If not set, the min iops will be set default value by observer.')
-        self.parser.add_option('--log_disk_size', type='string', help='The log disk size of the resource unit config.If not set, the log disk size will be set default value by observer.')
+        self.parser.add_option('-k', '--kms-encrypt-info', type='string', help='The KMS encryption information.')
+        self.parser.add_option('--memory-size', type='string', help='The memory size of the resource unit config')
+        self.parser.add_option('--max-cpu', type='int', help='The max cpu of the resource unit config, should be greater than 1.')
+        self.parser.add_option('--min-cpu', type='int', help='The min cpu of the resource unit config, should be greater than 1.If not set, the min cpu will be set to the max cpu.')
+        self.parser.add_option('--max-iops', type='int', help='The max iops of the resource unit config.If not set, the max iops will be set default value by observer.')
+        self.parser.add_option('--min-iops', type='int', help='The min iops of the resource unit config.If not set, the min iops will be set default value by observer.')
+        self.parser.add_option('--log-disk-size', type='string', help='The log disk size of the resource unit config.If not set, the log disk size will be set default value by observer.')
 
     def init(self, cmd, args):
         super(ClusterTenantRestoreCommand, self).init(cmd, args)
@@ -1660,6 +1671,7 @@ def __init__(self):
         self.register_command(ClusterRedeployCommand())
         self.register_command(ClusterEditConfigCommand())
         self.register_command(ClusterReloadCommand())
+        self.register_command(ClusterReloadCommand())
         self.register_command(CLusterUpgradeCommand())
         self.register_command(ClusterChangeRepositoryCommand())
         self.register_command(ClusterTenantCommand())
@@ -2656,18 +2668,22 @@ def __init__(self):
 
     def init(self, cmd, args):
         super(HostPrecheckCommand, self).init(cmd, args)
-        self.parser.set_usage('%s <ssh username>  <server ip>' % self.prev_cmd)
+        self.parser.set_usage('%s [<ssh username>] [<server ip>]' % self.prev_cmd)
         return self
 
     def _do_command(self, obd):
-        if len(self.cmds) == 2:
-            return obd.precheck_host(self.cmds[0], self.cmds[1])
-        elif len(self.cmds) == 3:
-            if self.cmds[2] == 'code100':
-                cmd_ret = obd.precheck_host(self.cmds[0], self.cmds[1], dev=True)
-                if isinstance(cmd_ret, bool):
-                    return cmd_ret
-                return CmdReturn(cmd_ret, cmd_ret)
+        # Set default values: username = current user, ip = 127.0.0.1
+        username = getpass.getuser() if len(self.cmds) == 0 else self.cmds[0]
+        ip = '127.0.0.1' if len(self.cmds) <= 1 else self.cmds[1]
+        
+        if len(self.cmds) == 3 and self.cmds[2] == 'code100':
+            cmd_ret = obd.precheck_host(username, ip, dev=True)
+            if isinstance(cmd_ret, bool):
+                return cmd_ret
+            return CmdReturn(cmd_ret, cmd_ret)
+        elif len(self.cmds) <= 2:
+            # Normal case: 0, 1, or 2 arguments (defaults applied above)
+            return obd.precheck_host(username, ip)
         else:
             return self._show_help()
 
@@ -2682,18 +2698,22 @@ def __init__(self):
 
     def init(self, cmd, args):
         super(HostInitCommand, self).init(cmd, args)
-        self.parser.set_usage('%s <ssh username>  <server ip>' % self.prev_cmd)
+        self.parser.set_usage('%s [<ssh username>] [<server ip>]' % self.prev_cmd)
         return self
 
     def _do_command(self, obd):
-        if len(self.cmds) == 2:
-            return obd.init_host(self.cmds[0], self.cmds[1])
-        elif len(self.cmds) == 3:
-            if self.cmds[2] == 'code101':
-                cmd_ret = obd.init_host(self.cmds[0], self.cmds[1], dev=True)
-                if isinstance(cmd_ret, bool):
-                    return cmd_ret
-                return CmdReturn(cmd_ret, cmd_ret)
+        # Set default values: username = current user, ip = 127.0.0.1
+        username = getpass.getuser() if len(self.cmds) == 0 else self.cmds[0]
+        ip = '127.0.0.1' if len(self.cmds) <= 1 else self.cmds[1]
+        
+        if len(self.cmds) == 3 and self.cmds[2] == 'code101':
+            cmd_ret = obd.init_host(username, ip, dev=True)
+            if isinstance(cmd_ret, bool):
+                return cmd_ret
+            return CmdReturn(cmd_ret, cmd_ret)
+        elif len(self.cmds) <= 2:
+            # Normal case: 0, 1, or 2 arguments (defaults applied above)
+            return obd.init_host(username, ip)
         else:
             return self._show_help()
 
@@ -2968,8 +2988,6 @@ def __init__(self):
 
     def _do_command(self, obd):
         if self.cmds:
-            if not getattr(self.opts, 'with_parameter', False):
-                setattr(self.opts, 'without_parameter', True)
             obd.set_options(self.opts)
             res = obd.restart_cluster(self.cmds[0])
             self.background_telemetry_task(obd)

_deploy.py

@@ -482,7 +482,12 @@ def _from_cluster_config(cls, conf, cluster_config):
 class ClusterConfig(object):
 
     def __init__(self, servers, name, version, tag, release, package_hash, comp_type, image_name, parser=None):
-        version = Version('1.0.0') if comp_type == 'docker' else version
+        if comp_type == 'docker':
+            version = Version('1.0.0')
+        if comp_type == 'dockers':
+            version = Version(tag)
+            if name == const.COMP_PRAG:
+                image_name = const.POWERRAG_IMAGE
         self._version = version
         self.origin_version = version
         self.tag = tag
@@ -595,6 +600,10 @@ def set_base_dir(self, base_dir):
     def docker_install(self):
         return self.comp_type == 'docker'
 
+    @property
+    def docker_compose_install(self):
+        return self.comp_type == 'dockers'
+
     @property
     def deploy_name(self):
         return self._deploy_config.name
@@ -1835,7 +1844,7 @@ def deploy_info(self):
     def _load_deploy_config(self, path):
         yaml_loader = YamlLoader(stdio=self.stdio)
         inner_config = InnerConfig(self.get_inner_config_path(self.config_dir), yaml_loader=yaml_loader)
-        deploy_config = DeployConfig(path, yaml_loader=yaml_loader, inner_config=inner_config, config_parser_manager=self.config_parser_manager, config_encrypted=self._config_encrypted,stdio=self.stdio)
+        deploy_config = DeployConfig(path, yaml_loader=yaml_loader, inner_config=inner_config, config_parser_manager=self.config_parser_manager, config_encrypted=self._config_encrypted, stdio=self.stdio)
         deploy_info = self.deploy_info
         for component_name in deploy_info.components:
             if component_name not in deploy_config.components:

_errno.py

@@ -150,6 +150,7 @@ class InitDirFailedErrorMessage(object):
 WC_FAIL_TO_RESTART_OR_RELOAD_AFTER_SCALE_OUT = OBDErrorCodeTemplate(1022, 'The cluster has been scaled out, but encountered problems when reloading or restarting. Details:\n{detail}')
 WC_CHANGE_SYSTEM_PARAMETER_FAILED = OBDErrorCodeTemplate(1023, '({server}) failed to change system parameter: {key})')
 
+
 # error code for observer
 EC_OBSERVER_NOT_ENOUGH_MEMORY = OBDErrorCodeTemplate(2000, '({ip}) not enough memory. (Free: {free}, Need: {need})')
 EC_OBSERVER_NOT_ENOUGH_MEMORY_ALAILABLE = OBDErrorCodeTemplate(2000, '({ip}) not enough memory. (Available [cat /proc/meminfo | grep MemAvailable] : {available}, Need: {need})')
@@ -182,7 +183,7 @@ class InitDirFailedErrorMessage(object):
 EC_OBSERVER_AUTO_START_DBUS_ENV = OBDErrorCodeTemplate(2019, "The current user does not have permission to enable autostart; sudo privileges are required and this operation is not allowed in container environments.")
 EC_MULTIPLE_NODES_SAME = OBDErrorCodeTemplate(2020, "Multiple observer nodes on the same server are not supported by the auto start feature")
 EC_OBSERVER_WEB_AUTO_START = OBDErrorCodeTemplate(2021, "Permission denied. Current user {user} on server {ip} requires sudo privileges.")
-
+EC_CPU_CORE_NOT_ENOUGH = OBDErrorCodeTemplate(2022, "({server}) Insufficient CPU cores. (Current: {current}, Required: {required})")
 
 WC_OBSERVER_SYS_MEM_TOO_LARGE = OBDErrorCodeTemplate(2010, '({server}): system_memory too large. system_memory should be less than {factor} * memory_limit/memory_limit_percentage.')
 
@@ -270,6 +271,13 @@ class InitDirFailedErrorMessage(object):
 EC_OMS_SERVER_CONNECT_METADB = OBDErrorCodeTemplate(4701, 'failed to connect meta db')
 EC_OMS_SERVER_CONNECT_INFLUXDB = OBDErrorCodeTemplate(4702, 'failed to connect influxdb')
 EC_OMS_NOT_ENOUGH_DISK = OBDErrorCodeTemplate(4703, '({ip}) {disk} not enough disk space. (Need: {need})')
+EC_OMS_UPDATE_NOT_DISABLE_HA = OBDErrorCodeTemplate(4704, 'HA is enabled, please disable it before upgrade.')
+
+#maas
+EC_MAAS_NOT_ENOUGH_DISK = OBDErrorCodeTemplate(4703, '({ip}) {disk} not enough disk space. (Need: {need})')
+
+#powerrag
+EC_POWERRAG_PROJECT_NAME_USED = OBDErrorCodeTemplate(4704, '({ip}) {project_name} is already in use')
 
 # sql
 EC_SQL_EXECUTE_FAILED = OBDErrorCodeTemplate(5000, "{sql} execute failed")

_mirror.py

@@ -60,7 +60,7 @@ def get_use_centos_release(stdio=None):
         'uos': {'20': 8},
         'anolis': {'23': 7},
         'openEuler': {'22.03': 7},
-        'kylin': {'V10': 8},
+        'kylin': {'V10': 7},
         'alinux': {'2': 7, '3': 8}
     }
     _SERVER_VARS = {

_repository.py

@@ -333,11 +333,13 @@ def _parse_path(self):
         if not self.version:
             self.set_version(version)
 
-    def _dump(self):
+    def _dump(self, **kwargs):
         data = {'version': self.version, 'hash': self.hash,
                 'release': self.release, 'arch': self.arch, 'size': self.size}
         if self.install_time:
             data['install_time'] = self.install_time
+        for k, v in kwargs.items():
+            data[k] = v
         try:
             with open(self.data_file_path, 'w') as f:
                 YamlLoader().dump(data, f)
@@ -461,6 +463,9 @@ def load_image(self, version, md5=None):
         self.install_time = time.time()
         self._dump()
 
+    def update_info(self, **kwargs):
+        self._dump(**kwargs)
+
     def clear(self):
         if os.path.exists(self.repository_dir):
             return DirectoryUtil.rm(self.repository_dir, self.stdio) and DirectoryUtil.mkdir(self.repository_dir, stdio=self.stdio)

_workflow.py

@@ -255,6 +255,8 @@ def __init__(self, home_path, component_name, workflow_name=None, dev_mode=False
     def get_workflow_template(self, version):
         template = self.get_best_plugin(version)
         if not template:
+            if not version:
+                version = '0.1'
             template = self._general_loader.get_best_plugin(version)
         return template
 

const.py

@@ -106,6 +106,14 @@
 DCDTC_SCRIPT = 'docker_copy_dumpfile_to_container.sh'
 DCDR_SCRIPT = 'docker_copy_dumpfile_rollback.sh'
 
+#maas
+COMP_MAAS = 'maas'
+
+# powerrag
+COMP_PRAG = 'powerrag'
+POWERRAG_IMAGE = 'harbor.oceanbase-dev.com/powerrag/powerrag'
+POWERRAG_PKG_HOME = 'POWERRAG_PKG_HOME'
+
 # service docs url
 DISABLE_SWAGGER = '<DISABLE_SWAGGER>'
 
@@ -168,4 +176,7 @@
         "type": "list",
         "sub_keys": ["cm_nodes", "cm_location", "cm_region"]
      }
-]
+]
+
+# only single server component
+SINGLE_SERVER_COMPONENTS = [COMP_PRAG, COMP_MAAS]