Skip to content

Commit 8daf22f

Browse files
tangledbytestangledbytes
committed
add support for uls deletion
Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> add tests Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> refactor Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> add support for flag in update command as well Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> add help message for update command Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> (cherry picked from commit c849216)
1 parent 35d1621 commit 8daf22f

File tree

6 files changed

+106
-27
lines changed

6 files changed

+106
-27
lines changed

‎src/cmd/manage_nsfs.js‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,11 @@ async function fetch_bucket_data(action, user_input) {
137137
data.s3_policy = user_input.bucket_policy;
138138
}
139139
}
140+
if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) {
141+
if (user_input.should_create_underlying_storage !== undefined) {
142+
data.should_create_underlying_storage = Boolean(user_input.should_create_underlying_storage);
143+
}
144+
}
140145
if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
141146
// @ts-ignore
142147
data = _.omitBy(data, _.isUndefined);
@@ -210,9 +215,20 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) {
210215
async function add_bucket(data) {
211216
data._id = mongo_utils.mongoObjectId();
212217
const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
213-
await set_bucker_owner(parsed_bucket_data);
218+
219+
const account = await account_id_cache.get_with_cache({ _id: parsed_bucket_data.owner_account, config_fs });
220+
await set_bucker_owner(parsed_bucket_data, account);
214221

215222
const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag);
223+
if (parsed_bucket_data.should_create_underlying_storage) {
224+
await BucketSpaceFS._create_uls(
225+
config_fs.fs_context,
226+
await native_fs_utils.get_fs_context(account.nsfs_account_config, parsed_bucket_data.fs_backend),
227+
data.name,
228+
data.path,
229+
config_fs.get_bucket_path_by_name(data.name)
230+
);
231+
}
216232

217233
return {
218234
code: ManageCLIResponse.BucketCreated,
@@ -284,6 +300,13 @@ async function delete_bucket(data, force) {
284300

285301
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
286302
await config_fs.delete_bucket_config_file(data.name);
303+
if (data.should_create_underlying_storage) {
304+
try {
305+
await nb_native().fs.rmdir(config_fs.fs_context, data.path);
306+
} catch (error) {
307+
dbg.warn('failed to delete underlying storage directory:', error);
308+
}
309+
}
287310
return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
288311
} catch (err) {
289312
if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
@@ -754,11 +777,13 @@ function get_access_keys(action, user_input) {
754777
/**
755778
* set_bucker_owner gets bucket owner from cache by its id and sets bucket_owner name on the bucket data
756779
* @param {object} bucket_data
780+
* @param {*} [account_data]
757781
*/
758-
async function set_bucker_owner(bucket_data) {
759-
let account_data;
782+
async function set_bucker_owner(bucket_data, account_data) {
760783
try {
761-
account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
784+
if (!account_data) {
785+
account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
786+
}
762787
} catch (err) {
763788
dbg.warn(`set_bucker_owner.couldn't find bucket owner data by id ${bucket_data.owner_account}`);
764789
}

‎src/manage_nsfs/manage_nsfs_constants.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
6161
};
6262

6363
const VALID_OPTIONS_BUCKET = {
64-
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65-
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
64+
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', 'should_create_underlying_storage', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65+
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', 'should_create_underlying_storage', ...CLI_MUTUAL_OPTIONS]),
6666
'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
6767
'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
6868
'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -142,6 +142,7 @@ const OPTION_TYPE = {
142142
ips: 'string',
143143
force: 'boolean',
144144
anonymous: 'boolean',
145+
should_create_underlying_storage: 'boolean',
145146
// health options
146147
deployment_type: 'string',
147148
all_account_details: 'boolean',

‎src/manage_nsfs/manage_nsfs_help_utils.js‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ Flags:
243243
--fs_backend <none | GPFS | CEPH_FS | NFSv4> (optional) Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND)
244244
--force_md5_etag <true | false> (optional) Set the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
245245
--from_file <string> (optional) Use details from the JSON file, there is no need to mention all the properties individually in the CLI
246-
246+
--should_create_underlying_storage <true | false> (optional) Force creation of bucket's underlying storage directory
247247
`;
248248

249249
const BUCKET_FLAGS_UPDATE = `
@@ -264,6 +264,7 @@ Flags:
264264
--bucket_policy <string> (optional) Update the bucket policy, type is a string of valid JSON policy (unset with '')
265265
--fs_backend <none | GPFS | CEPH_FS | NFSv4> (optional) Update the filesystem type (unset with '') (default config.NSFS_NC_STORAGE_BACKEND)
266266
--force_md5_etag <true | false> (optional) Update the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
267+
--should_create_underlying_storage <true | false> (optional) Update the bucket to manage the underlying storage
267268
268269
`;
269270

‎src/manage_nsfs/manage_nsfs_validations.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ async function validate_bucket_args(config_fs, data, action) {
408408
await check_new_name_exists(TYPES.BUCKET, config_fs, action, data);
409409
// in case we have the fs_backend it changes the fs_context that we use for the path
410410
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
411-
if (!config.NC_DISABLE_ACCESS_CHECK) {
411+
if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
412412
const exists = await native_fs_utils.is_path_exists(fs_context_fs_backend, data.path);
413413
if (!exists) {
414414
throw_cli_error(ManageCLIError.InvalidStoragePath, data.path);
@@ -420,7 +420,7 @@ async function validate_bucket_args(config_fs, data, action) {
420420

421421
const account_fs_context = await native_fs_utils.get_fs_context(owner_account_data.nsfs_account_config,
422422
owner_account_data.nsfs_account_config.fs_backend);
423-
if (!config.NC_DISABLE_ACCESS_CHECK) {
423+
if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
424424
const accessible = await native_fs_utils.is_dir_accessible(account_fs_context, data.path);
425425
if (!accessible) {
426426
throw_cli_error(ManageCLIError.InaccessibleStoragePath, data.path);

‎src/sdk/bucketspace_fs.js‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -324,24 +324,16 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
324324
}
325325

326326
// create bucket's underlying storage directory
327-
try {
328-
await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR));
329-
const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
330-
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
331-
if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
332-
return curr;
333-
}, {});
334-
335-
new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
336-
...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
337-
});
338-
} catch (err) {
339-
dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err);
340-
new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
341-
.create_event(name, { bucket: name, path: bucket_storage_path }, err);
342-
await nb_native().fs.unlink(this.fs_context, bucket_config_path);
343-
throw translate_error_codes(err, entity_enum.BUCKET);
344-
}
327+
await BucketSpaceFS._create_uls(this.fs_context, fs_context, name, bucket_storage_path, bucket_config_path);
328+
const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
329+
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
330+
if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
331+
return curr;
332+
}, {});
333+
334+
new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
335+
...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
336+
});
345337
});
346338
}
347339

@@ -1036,6 +1028,26 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
10361028
static _objectify_tagging_arr(tagging) {
10371029
return (tagging || []).reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {});
10381030
}
1031+
1032+
/**
1033+
* _create_uls creates underylying storage at the given storage_path
1034+
* @param {*} fs_context - root fs_context
1035+
* @param {*} acc_fs_context - fs_context associated with the performing account
1036+
* @param {*} name - bucket name
1037+
* @param {*} storage_path - bucket's storage path
1038+
* @param {*} cfg_path - bucket's configuration path
1039+
*/
1040+
static async _create_uls(fs_context, acc_fs_context, name, storage_path, cfg_path) {
1041+
try {
1042+
await nb_native().fs.mkdir(acc_fs_context, storage_path, get_umasked_mode(config.BASE_MODE_DIR));
1043+
} catch (error) {
1044+
dbg.error('BucketSpaceFS: _create_uls could not create underlying directory - nsfs, deleting bucket', error);
1045+
new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
1046+
.create_event(name, { bucket: name, path: storage_path }, error);
1047+
await nb_native().fs.unlink(fs_context, cfg_path);
1048+
throw translate_error_codes(error, entity_enum.BUCKET);
1049+
}
1050+
}
10391051
}
10401052

10411053
module.exports = BucketSpaceFS;

‎src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,26 @@ describe('manage nsfs cli bucket flow', () => {
265265
expect(JSON.parse(res_list).response.reply.map(item => item.name))
266266
.toEqual(expect.arrayContaining([bucket_options.name]));
267267
});
268+
269+
it('cli create bucket - with ULS true', async () => {
270+
await fs_utils.create_fresh_path(root_path);
271+
await set_path_permissions_and_owner(root_path, account_defaults, 0o700);
272+
273+
const action = ACTIONS.ADD;
274+
const bucket_options = { config_root, ...bucket_defaults, should_create_underlying_storage: true };
275+
await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
276+
await fs_utils.file_must_exist(bucket_options.path);
277+
278+
const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
279+
expect(bucket).toBeDefined();
280+
expect(bucket.name).toBe(bucket_options.name);
281+
expect(bucket.should_create_underlying_storage).toBe(true);
282+
283+
const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.STATUS, { name: bucket_options.name, config_root });
284+
const parsed_res = JSON.parse(res);
285+
expect(parsed_res.response.reply.name).toBe(bucket_options.name);
286+
expect(parsed_res.response.reply.should_create_underlying_storage).toBe(true);
287+
});
268288
});
269289

270290
describe('cli create bucket using from_file', () => {
@@ -806,6 +826,26 @@ describe('manage nsfs cli bucket flow', () => {
806826
const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
807827
expect(is_bucket_exists).toBe(false);
808828
});
829+
830+
831+
it('cli delete bucket when should_create_underlying_storage is true', async () => {
832+
let path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
833+
expect(path_exists).toBe(true);
834+
835+
const bucket_options = { config_root, name: 'bucket1' };
836+
837+
const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
838+
await config_fs.update_bucket_config_file({ ...bucket, should_create_underlying_storage: true });
839+
840+
const action = ACTIONS.DELETE;
841+
const res = await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
842+
expect(JSON.parse(res.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code);
843+
const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
844+
expect(is_bucket_exists).toBe(false);
845+
846+
path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
847+
expect(path_exists).toBe(false);
848+
});
809849
});
810850

811851
describe('cli status bucket ', () => {

0 commit comments

Comments
 (0)