Skip to content

Commit 14f4d28

Browse files
tangledbytestangledbytes
committed
add support for uls deletion
Signed-off-by: Utkarsh Srivastava <[email protected]> add tests Signed-off-by: Utkarsh Srivastava <[email protected]> refactor Signed-off-by: Utkarsh Srivastava <[email protected]> add support for flag in update command as well Signed-off-by: Utkarsh Srivastava <[email protected]>
1 parent c42c9ac commit 14f4d28

File tree

6 files changed

+105
-27
lines changed

6 files changed

+105
-27
lines changed

‎src/cmd/manage_nsfs.js‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,11 @@ async function fetch_bucket_data(action, user_input) {
149149
data.s3_policy = user_input.bucket_policy;
150150
}
151151
}
152+
if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) {
153+
if (user_input.should_create_underlying_storage !== undefined) {
154+
data.should_create_underlying_storage = Boolean(user_input.should_create_underlying_storage);
155+
}
156+
}
152157
if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
153158
// @ts-ignore
154159
data = _.omitBy(data, _.isUndefined);
@@ -221,9 +226,20 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) {
221226
async function add_bucket(data) {
222227
data._id = mongo_utils.mongoObjectId();
223228
const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
224-
await set_bucker_owner(parsed_bucket_data);
229+
230+
const account = await account_id_cache.get_with_cache({ _id: parsed_bucket_data.owner_account, config_fs });
231+
await set_bucker_owner(parsed_bucket_data, account);
225232

226233
const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag);
234+
if (parsed_bucket_data.should_create_underlying_storage) {
235+
await BucketSpaceFS._create_uls(
236+
config_fs.fs_context,
237+
await native_fs_utils.get_fs_context(account.nsfs_account_config, parsed_bucket_data.fs_backend),
238+
data.name,
239+
data.path,
240+
config_fs.get_bucket_path_by_name(data.name)
241+
);
242+
}
227243

228244
return {
229245
code: ManageCLIResponse.BucketCreated,
@@ -295,6 +311,13 @@ async function delete_bucket(data, force) {
295311

296312
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
297313
await config_fs.delete_bucket_config_file(data.name);
314+
if (data.should_create_underlying_storage) {
315+
try {
316+
await nb_native().fs.rmdir(config_fs.fs_context, data.path);
317+
} catch (error) {
318+
dbg.warn('failed to delete underlying storage directory:', error);
319+
}
320+
}
298321
return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
299322
} catch (err) {
300323
if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
@@ -784,11 +807,13 @@ function get_access_keys(action, user_input) {
784807
/**
785808
* set_bucker_owner gets bucket owner from cache by its id and sets bucket_owner name on the bucket data
786809
* @param {object} bucket_data
810+
* @param {*} [account_data]
787811
*/
788-
async function set_bucker_owner(bucket_data) {
789-
let account_data;
812+
async function set_bucker_owner(bucket_data, account_data) {
790813
try {
791-
account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
814+
if (!account_data) {
815+
account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
816+
}
792817
} catch (err) {
793818
dbg.warn(`set_bucker_owner.couldn't find bucket owner data by id ${bucket_data.owner_account}`);
794819
}

‎src/manage_nsfs/manage_nsfs_constants.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
6161
};
6262

6363
const VALID_OPTIONS_BUCKET = {
64-
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65-
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
64+
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', 'should_create_underlying_storage', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65+
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', 'should_create_underlying_storage', ...CLI_MUTUAL_OPTIONS]),
6666
'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
6767
'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
6868
'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -143,6 +143,7 @@ const OPTION_TYPE = {
143143
force: 'boolean',
144144
anonymous: 'boolean',
145145
default_connection: 'string',
146+
should_create_underlying_storage: 'boolean',
146147
// health options
147148
deployment_type: 'string',
148149
all_account_details: 'boolean',

‎src/manage_nsfs/manage_nsfs_help_utils.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ Flags:
243243
--fs_backend <none | GPFS | CEPH_FS | NFSv4> (optional) Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND)
244244
--force_md5_etag <true | false> (optional) Set the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
245245
--from_file <string> (optional) Use details from the JSON file, there is no need to mention all the properties individually in the CLI
246-
246+
--should_create_underlying_storage <true | false> (optional) Force creation of bucket's underlying storage directory
247247
`;
248248

249249
const BUCKET_FLAGS_UPDATE = `

‎src/manage_nsfs/manage_nsfs_validations.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ async function validate_bucket_args(config_fs, data, action) {
455455
await check_new_name_exists(TYPES.BUCKET, config_fs, action, data);
456456
// in case we have the fs_backend it changes the fs_context that we use for the path
457457
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
458-
if (!config.NC_DISABLE_ACCESS_CHECK) {
458+
if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
459459
const exists = await native_fs_utils.is_path_exists(fs_context_fs_backend, data.path);
460460
if (!exists) {
461461
throw_cli_error(ManageCLIError.InvalidStoragePath, data.path);
@@ -467,7 +467,7 @@ async function validate_bucket_args(config_fs, data, action) {
467467

468468
const account_fs_context = await native_fs_utils.get_fs_context(owner_account_data.nsfs_account_config,
469469
owner_account_data.nsfs_account_config.fs_backend);
470-
if (!config.NC_DISABLE_ACCESS_CHECK) {
470+
if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
471471
const accessible = await native_fs_utils.is_dir_accessible(account_fs_context, data.path);
472472
if (!accessible) {
473473
throw_cli_error(ManageCLIError.InaccessibleStoragePath, data.path);

‎src/sdk/bucketspace_fs.js‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -324,24 +324,16 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
324324
}
325325

326326
// create bucket's underlying storage directory
327-
try {
328-
await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR));
329-
const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
330-
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
331-
if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
332-
return curr;
333-
}, {});
334-
335-
new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
336-
...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
337-
});
338-
} catch (err) {
339-
dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err);
340-
new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
341-
.create_event(name, { bucket: name, path: bucket_storage_path }, err);
342-
await nb_native().fs.unlink(this.fs_context, bucket_config_path);
343-
throw translate_error_codes(err, entity_enum.BUCKET);
344-
}
327+
await BucketSpaceFS._create_uls(this.fs_context, fs_context, name, bucket_storage_path, bucket_config_path);
328+
const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
329+
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
330+
if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
331+
return curr;
332+
}, {});
333+
334+
new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
335+
...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
336+
});
345337
});
346338
}
347339

@@ -1094,6 +1086,26 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
10941086
static _objectify_tagging_arr(tagging) {
10951087
return (tagging || []).reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {});
10961088
}
1089+
1090+
/**
1091+
* _create_uls creates underylying storage at the given storage_path
1092+
* @param {*} fs_context - root fs_context
1093+
* @param {*} acc_fs_context - fs_context associated with the performing account
1094+
* @param {*} name - bucket name
1095+
* @param {*} storage_path - bucket's storage path
1096+
* @param {*} cfg_path - bucket's configuration path
1097+
*/
1098+
static async _create_uls(fs_context, acc_fs_context, name, storage_path, cfg_path) {
1099+
try {
1100+
await nb_native().fs.mkdir(acc_fs_context, storage_path, get_umasked_mode(config.BASE_MODE_DIR));
1101+
} catch (error) {
1102+
dbg.error('BucketSpaceFS: _create_uls could not create underlying directory - nsfs, deleting bucket', error);
1103+
new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
1104+
.create_event(name, { bucket: name, path: storage_path }, error);
1105+
await nb_native().fs.unlink(fs_context, cfg_path);
1106+
throw translate_error_codes(error, entity_enum.BUCKET);
1107+
}
1108+
}
10971109
}
10981110

10991111
module.exports = BucketSpaceFS;

‎src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,26 @@ describe('manage nsfs cli bucket flow', () => {
265265
expect(JSON.parse(res_list).response.reply.map(item => item.name))
266266
.toEqual(expect.arrayContaining([bucket_options.name]));
267267
});
268+
269+
it('cli create bucket - with ULS true', async () => {
270+
await fs_utils.create_fresh_path(root_path);
271+
await set_path_permissions_and_owner(root_path, account_defaults, 0o700);
272+
273+
const action = ACTIONS.ADD;
274+
const bucket_options = { config_root, ...bucket_defaults, should_create_underlying_storage: true };
275+
await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
276+
await fs_utils.file_must_exist(bucket_options.path);
277+
278+
const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
279+
expect(bucket).toBeDefined();
280+
expect(bucket.name).toBe(bucket_options.name);
281+
expect(bucket.should_create_underlying_storage).toBe(true);
282+
283+
const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.STATUS, { name: bucket_options.name, config_root });
284+
const parsed_res = JSON.parse(res);
285+
expect(parsed_res.response.reply.name).toBe(bucket_options.name);
286+
expect(parsed_res.response.reply.should_create_underlying_storage).toBe(true);
287+
});
268288
});
269289

270290
describe('cli create bucket using from_file', () => {
@@ -805,6 +825,26 @@ describe('manage nsfs cli bucket flow', () => {
805825
const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
806826
expect(is_bucket_exists).toBe(false);
807827
});
828+
829+
830+
it('cli delete bucket when should_create_underlying_storage is true', async () => {
831+
let path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
832+
expect(path_exists).toBe(true);
833+
834+
const bucket_options = { config_root, name: 'bucket1' };
835+
836+
const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
837+
await config_fs.update_bucket_config_file({ ...bucket, should_create_underlying_storage: true });
838+
839+
const action = ACTIONS.DELETE;
840+
const res = await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
841+
expect(JSON.parse(res.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code);
842+
const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
843+
expect(is_bucket_exists).toBe(false);
844+
845+
path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
846+
expect(path_exists).toBe(false);
847+
});
808848
});
809849

810850
describe('cli status bucket ', () => {

0 commit comments

Comments
 (0)