mid eval

Author: tstamler

examples/python/remote_storage_example/agent_file.in

@@ -1 +1 @@
-target1 127.0.0.1 8888
+target1 10.52.52.47 8888

examples/python/remote_storage_example/client_command.sh

@@ -9,4 +9,6 @@
 # 1048576
 # 524288
 
-python3 nixl_p2p_storage_example.py --fileprefix /raid/scratch/testfiles/client --role client --agents_file agent_file.in --name client1 --batch_size 32 --buf_size 524288
+export CUFILE_ENV_PATH_JSON="/workspace/nixl-tim/examples/python/remote_storage_example/cufile.json"
+
+python3 nixl_p2p_storage_example.py --fileprefix /raid/scratch/testfiles/client --role client --agents_file agent_file.in --name client1 --batch_size 128 --buf_size 2097152

examples/python/remote_storage_example/cufile.json

@@ -0,0 +1,24 @@
+{
+    // NOTE : Application can override custom configuration via export CUFILE_ENV_PATH_JSON=<filepath>
+    // e.g : export CUFILE_ENV_PATH_JSON="/home/<xxx>/cufile.json"
+
+            "properties": {
+                            // allow compat mode, this will enable use of cuFile posix read/writes
+                            "allow_compat_mode": true,
+                            // max IO chunk size (parameter should be multiples of 64K) used by cuFileRead/Write internally per IO request
+                            "max_direct_io_size_kb" : 16384,
+                            // device memory size (parameter should be 4K aligned) for reserving bounce buffers for the entire GPU
+                            "max_device_cache_size_kb" : 2097152,
+                            // Note: ensure (max_device_cache_size_kb / per_buffer_cache_size_kb) >= io_batchsize
+                            "per_buffer_cache_size_kb": 16384,
+                            // limit on maximum device memory size (parameter should be 4K aligned) that can be pinned for a given process
+                            "max_device_pinned_mem_size_kb" : 33554432,
+                            // per-io bounce-buffer size (parameter should be multiples of 64K) ranging from 1024kb to 16384kb
+                            "per_buffer_cache_size_kb" : 16384,
+
+                            // posix bounce buffer pool size allocations
+                            "posix_pool_slab_size_kb" : [ 4, 1024, 16384],
+                            // posix bounce buffer pool max counts
+                            "posix_pool_slab_count": [512, 512, 512]
+            }
+}

examples/python/remote_storage_example/nixl_p2p_storage_example.py

@@ -52,18 +52,16 @@ def remote_storage_transfer(my_agent, my_mem_descs, operation, remote_agent_name
     logger.info(f"Sending {operation} request to {remote_agent_name}")
     test_descs_str = my_agent.get_serialized_descs(my_mem_descs)
 
-    start_time = time.perf_counter()
+    start_time = time.time()
 
     my_agent.send_notif(remote_agent_name, operation + iterations + test_descs_str)
 
     while not my_agent.check_remote_xfer_done(remote_agent_name, b"COMPLETE"):
         continue
 
-    end_time = time.perf_counter()
+    end_time = time.time()
 
-    elapsed = end_time - start_time
-
-    logger.info(f"Time for {iterations} iterations: {elapsed} seconds")
+    logger.info(f"Time for {iterations} iterations: {end_time - start_time} seconds")
 
 
 def connect_to_agents(my_agent, agents_file):
@@ -193,7 +191,7 @@ def run_client(my_agent, nixl_mem_reg_descs, nixl_file_reg_descs, agents_file):
     # For sample purposes, write to and then read from local storage
     logger.info("Starting local transfer test...")
 
-    start_time = time.perf_counter()
+    start_time = time.time()
 
     for i in range (1, 100):
         execute_transfer(
@@ -206,13 +204,13 @@ def run_client(my_agent, nixl_mem_reg_descs, nixl_file_reg_descs, agents_file):
             ["GDS_MT"]
         )
 
-    end_time = time.perf_counter()
+    end_time = time.time()
 
     elapsed = end_time - start_time
 
     logger.info(f"Time for 100 WRITE iterations: {elapsed} seconds")
 
-    start_time = time.perf_counter()
+    start_time = time.time()
 
     for i in range (1, 100):
         execute_transfer(
@@ -225,7 +223,7 @@ def run_client(my_agent, nixl_mem_reg_descs, nixl_file_reg_descs, agents_file):
             ["GDS_MT"]
         )
 
-    end_time = time.perf_counter()
+    end_time = time.time()
 
     elapsed = end_time - start_time
 

examples/python/remote_storage_example/nixl_storage_utils/__pycache__/common.cpython-310.pyc

@@ -38,7 +38,7 @@ def create_agent_with_plugins(agent_name, port):
     plugin_list = new_nixl_agent.get_plugin_list()
 
     if "GDS_MT" in plugin_list:
-        new_nixl_agent.create_backend("GDS_MT", {"thread_count": "8"})
+        new_nixl_agent.create_backend("GDS_MT")
         logger.info("Using GDS storage backend")
     if "POSIX" in plugin_list:
         new_nixl_agent.create_backend("POSIX")

examples/python/remote_storage_example/nixl_storage_utils/common.py

@@ -9,4 +9,6 @@
 # 1048576
 # 524288
 
-python3 nixl_p2p_storage_example.py --fileprefix /raid/scratch/testfiles/target1 --role server --port 8888 --name target1 --batch_size 32 --buf_size 524288
+export UCX_MAX_RMA_RAILS=1
+
+python3 nixl_p2p_storage_example.py --fileprefix /raid/scratch/testfiles/target1 --role server --port 8888 --name target1 --batch_size 32 --buf_size 8388608