NSFS | Improve list objects performance on top of NS FS

Signed-off-by: naveenpaul1 <[email protected]>

Author: naveenpaul1

config.js

@@ -843,6 +843,9 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;
 // of `manage_nsfs glacier restore`
 config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;
 
+// enable/disable unsorted listing application level
+config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false;
+
 // NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies
 // when NooBaa should allow running glacier expiry process
 // NOTE: This will also be in the same timezone as specified in

src/api/object_api.js

@@ -723,6 +723,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {
@@ -777,6 +780,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {

src/endpoint/s3/ops/s3_get_bucket.js

@@ -41,6 +41,7 @@ async function get_bucket(req) {
         bucket: req.params.bucket,
         prefix: req.query.prefix,
         delimiter: req.query.delimiter,
+        list_type: list_type,
         limit: Math.min(max_keys_received, 1000),
         key_marker: list_type === '2' ?
             (s3_utils.cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,

src/native/fs/fs_napi.cpp

@@ -2105,7 +2105,7 @@ struct TellDir : public FSWrapWorker<DirWrap>
     }
     virtual void OnOK()
     {
-        DBG0("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
+        DBG1("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
         Napi::Env env = Env();
         auto res = Napi::BigInt::New(env, _tell_res);
         _deferred.Resolve(res);

src/sdk/keymarker_fs.js

@@ -0,0 +1,50 @@
+/* Copyright (C) 2020 NooBaa */
+'use strict';
+
+class KeyMarkerFS {
+    constructor({ marker, marker_pos, pre_dir, pre_dir_pos }, is_unsorted = false) {
+        this.marker = marker;
+        this.marker_pos = marker_pos.toString();
+        this.pre_dir = pre_dir;
+        this.pre_dir_pos = pre_dir_pos;
+        this.key_marker_value = marker;
+        this.current_dir = '';
+        this.is_unsorted = is_unsorted;
+        this.last_pre_dir = '';
+        this.last_pre_dir_position = '';
+        if (is_unsorted) {
+            this.current_dir = pre_dir.length > 0 && marker.includes('/') ?
+                            marker.substring(0, marker.lastIndexOf('/') + 1) : '';
+        }
+    }
+
+    async update_key_marker(marker, marker_pos) {
+        this.marker = marker;
+        this.marker_pos = marker_pos;
+        this.key_marker_value = marker;
+    }
+
+    async get_previour_dir_length() {
+        return this.pre_dir.length;
+    }
+
+    async get_previour_dir_info() {
+        return {
+            pre_dir_path: this.pre_dir.pop(),
+            pre_dir_position: this.pre_dir_pos.pop(),
+        };
+    }
+
+    async add_previour_dir(pre_dir, pre_dir_pos) {
+        this.pre_dir.push(pre_dir);
+        this.pre_dir_pos.push(pre_dir_pos.toString());
+    }
+
+    async update_last_previour_dir(last_pre_dir, last_pre_dir_position) {
+        this.last_pre_dir = last_pre_dir;
+        this.last_pre_dir_position = last_pre_dir_position;
+    }
+}
+
+// EXPORTS
+module.exports = KeyMarkerFS;

src/sdk/namespace_fs.js

@@ -491,6 +491,26 @@ function get_new_buckets_path_by_test_env(new_buckets_full_path, new_buckets_dir
     return is_nc_coretest ? path.join(new_buckets_full_path, new_buckets_dir) : new_buckets_dir;
 }
 
+
+/**
+ * common dummy SDK for testing
+ */
+function make_dummy_object_sdk() {
+    return {
+        requesting_account: {
+            force_md5_etag: false,
+            nsfs_account_config: {
+                uid: process.getuid(),
+                gid: process.getgid(),
+            }
+        },
+        abort_controller: new AbortController(),
+        throw_if_aborted() {
+            if (this.abort_controller.signal.aborted) throw new Error('request aborted signal');
+        }
+    };
+}
+
 /**
  * write_manual_config_file writes config file directly to the file system without using config FS
  * used for creating backward compatibility tests, invalid config files etc
@@ -758,10 +778,12 @@ exports.create_identity_dir_if_missing = create_identity_dir_if_missing;
 exports.symlink_account_name = symlink_account_name;
 exports.symlink_account_access_keys = symlink_account_access_keys;
 exports.create_file = create_file;
+exports.make_dummy_object_sdk = make_dummy_object_sdk;
 exports.create_redirect_file = create_redirect_file;
 exports.delete_redirect_file = delete_redirect_file;
 exports.create_system_json = create_system_json;
 exports.update_system_json = update_system_json;
 exports.fail_test_if_default_config_dir_exists = fail_test_if_default_config_dir_exists;
 exports.create_config_dir = create_config_dir;
 exports.clean_config_dir = clean_config_dir;
+