add support for uls deletion

Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>

Author: tangledbytes

src/cmd/manage_nsfs.js

@@ -131,7 +131,7 @@ async function fetch_bucket_data(action, user_input) {
         versioning: action === ACTIONS.ADD ? 'DISABLED' : undefined,
         creation_date: action === ACTIONS.ADD ? new Date().toISOString() : undefined,
         path: user_input.path,
-        should_create_underlying_storage: action === ACTIONS.ADD ? false : undefined,
+        should_create_underlying_storage: action === ACTIONS.ADD ? Boolean(user_input.should_create_underlying_storage) : undefined,
         new_name: user_input.new_name === undefined ? undefined : String(user_input.new_name),
         fs_backend: user_input.fs_backend === undefined ? config.NSFS_NC_STORAGE_BACKEND : String(user_input.fs_backend),
         force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
@@ -221,9 +221,25 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) {
 async function add_bucket(data) {
     data._id = mongo_utils.mongoObjectId();
     const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
-    await set_bucker_owner(parsed_bucket_data);
+
+    const account = await account_id_cache.get_with_cache({ _id: parsed_bucket_data.owner_account, config_fs });
+    await set_bucker_owner(parsed_bucket_data, account);
 
     const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag);
+    if (parsed_bucket_data.should_create_underlying_storage) {
+        try {
+            await nb_native().fs.mkdir(
+                await native_fs_utils.get_fs_context(account.nsfs_account_config, parsed_bucket_data.fs_backend),
+                data.path,
+                native_fs_utils.get_umasked_mode(config.BASE_MODE_DIR),
+            );
+        } catch (error) {
+            new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
+                .create_event(data.name, { bucket: data.name, path: data.path }, error);
+            await config_fs.delete_bucket_config_file(data.name);
+            throw error;
+        }
+    }
 
     return {
         code: ManageCLIResponse.BucketCreated,
@@ -295,6 +311,13 @@ async function delete_bucket(data, force) {
 
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
+        if (data.should_create_underlying_storage) {
+            try {
+                await nb_native().fs.rmdir(config_fs.fs_context, data.path);
+            } catch (error) {
+                dbg.warn('failed to delete underlying storage directory:', error);
+            }
+        }
         return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
     } catch (err) {
         if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
@@ -784,11 +807,13 @@ function get_access_keys(action, user_input) {
 /**
  * set_bucker_owner gets bucket owner from cache by its id and sets bucket_owner name on the bucket data
  * @param {object} bucket_data 
+ * @param {*} [account_data]
  */
-async function set_bucker_owner(bucket_data) {
-    let account_data;
+async function set_bucker_owner(bucket_data, account_data) {
     try {
-        account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
+        if (!account_data) {
+            account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
+        }
     } catch (err) {
         dbg.warn(`set_bucker_owner.couldn't find bucket owner data by id ${bucket_data.owner_account}`);
     }

src/manage_nsfs/manage_nsfs_constants.js

@@ -61,7 +61,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
 };
 
 const VALID_OPTIONS_BUCKET = {
-    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
+    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', 'should_create_underlying_storage', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
     'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
@@ -143,6 +143,7 @@ const OPTION_TYPE = {
     force: 'boolean',
     anonymous: 'boolean',
     default_connection: 'string',
+    should_create_underlying_storage: 'boolean',
     // health options
     deployment_type: 'string',
     all_account_details: 'boolean',

src/manage_nsfs/manage_nsfs_help_utils.js

@@ -243,7 +243,7 @@ Flags:
     --fs_backend <none | GPFS | CEPH_FS | NFSv4>          (optional)        Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND)
     --force_md5_etag <true | false>                       (optional)        Set the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
-
+    --should_create_underlying_storage <true | false>     (optional)        Force creation of bucket's underlying storage directory
 `;
 
 const BUCKET_FLAGS_UPDATE = `

src/manage_nsfs/manage_nsfs_validations.js

@@ -455,7 +455,7 @@ async function validate_bucket_args(config_fs, data, action) {
         await check_new_name_exists(TYPES.BUCKET, config_fs, action, data);
         // in case we have the fs_backend it changes the fs_context that we use for the path
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
-        if (!config.NC_DISABLE_ACCESS_CHECK) {
+        if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
             const exists = await native_fs_utils.is_path_exists(fs_context_fs_backend, data.path);
             if (!exists) {
                 throw_cli_error(ManageCLIError.InvalidStoragePath, data.path);
@@ -467,7 +467,7 @@ async function validate_bucket_args(config_fs, data, action) {
 
         const account_fs_context = await native_fs_utils.get_fs_context(owner_account_data.nsfs_account_config,
             owner_account_data.nsfs_account_config.fs_backend);
-        if (!config.NC_DISABLE_ACCESS_CHECK) {
+        if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
             const accessible = await native_fs_utils.is_dir_accessible(account_fs_context, data.path);
             if (!accessible) {
                 throw_cli_error(ManageCLIError.InaccessibleStoragePath, data.path);