Merge Release 2.4.0

.kitchen.cloud.yml

@@ -2,6 +2,7 @@
 driver_config:
   retryable_sleep: 15
   retryable_tries: 20
+  retry_limit: 6
   aws_ssh_key_id: <%= ENV['AWS_KEYPAIR_NAME'] %>
   region: <%= ENV['AWS_DEFAULT_REGION'] %>
   instance_type: <%= ENV['AWS_FLAVOR_ID'] %>
@@ -16,6 +17,8 @@ driver_config:
 provisioner:
   name: chef_zero
   require_chef_omnibus: 14.2.0
+  # use custom chef install URL to cope with issue https://github.com/chef/bento/issues/609
+  chef_omnibus_url: https://raw.githubusercontent.com/aws/aws-parallelcluster-cookbook/develop/util/chef-install.sh
   retry_on_exit_code:
     - 35 # 35 is the exit code signaling that the node is rebooting
   max_retries: 1

.kitchen.yml

@@ -68,11 +68,14 @@ suites:
         cfn_ephemeral_dir: <%= ENV['CFN_EPHEMERAL_DIR'] %>
         cfn_shared_dir: <%= ENV['CFN_SHARED_DIR'] %>
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: sge_config_MasterServer
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::sge_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -87,11 +90,14 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_ddb_table: <%= ENV['CFN_DDB_TABLE'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: torque_config_MasterServer
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::torque_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -106,11 +112,14 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_ddb_table: <%= ENV['CFN_DDB_TABLE'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: slurm_config_MasterServer
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::slurm_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -125,11 +134,14 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_ddb_table: <%= ENV['CFN_DDB_TABLE'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: sge_config_ComputeFleet
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::sge_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -144,11 +156,14 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_master: <%= ENV['CFN_MASTER'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: torque_config_ComputeFleet
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::torque_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -163,11 +178,14 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_master: <%= ENV['CFN_MASTER'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>
 
   - name: slurm_config_ComputeFleet
     run_list:
       - recipe[aws-parallelcluster::_prep_env]
       - recipe[aws-parallelcluster::slurm_config]
+      - recipe[aws-parallelcluster::finalize]
       - recipe[aws-parallelcluster::tests]
     attributes:
       cfncluster:
@@ -182,3 +200,5 @@ suites:
         cfn_cluster_user: <%= ENV['CFN_CLUSTER_USER'] %>
         cfn_sqs_queue: <%= ENV['CFN_SQS_QUEUE'] %>
         cfn_master: <%= ENV['CFN_MASTER'] %>
+        custom_node_package: <%= ENV['PARALLELCLUSTER_NODE_URL'] %>
+        os: <%= ENV['OS'] %>

CHANGELOG.md

@@ -3,6 +3,36 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
+2.4.0
+-----
+
+**ENHANCEMENTS**
+- Add support for EFA on Centos 7, Amazon Linux and Ubuntu 1604
+- Add support for Ubuntu in China region `cn-northwest-1`
+
+**CHANGES**
+- SGE: changed following parameters in global configuration
+  - `max_unheard 00:03:00`: allows a faster reaction in case of faulty nodes
+  - `reschedule_unknown 00:00:30`: enables rescheduling of jobs running on failing nodes
+  - `qmaster_params ENABLE_FORCED_QDEL_IF_UNKNOWN`: forces job deletion on unresponsive nodes
+  - `qmaster_params ENABLE_RESCHEDULE_KILL`: forces rescheduling or killing of jobs running on failing nodes
+- Slurm: decrease SlurmdTimeout to 120 seconds to speed up replacement of faulty nodes
+- Always use full master FQDN when mounting NFS on compute nodes. This solves some issues occurring with some networking
+  setups and custom DNS configurations
+- Set soft and hard ulimit on open files to 10000 for all supported OSs
+- Pin python `supervisor` version to 3.4.0
+- Remove unused `compute_instance_type` from jobwatcher.cfg
+- Removed unused `max_queue_size` from sqswatcher.cfg
+- Remove double quoting of the post_install args
+
+**BUG FIXES**
+- Fix issue that was preventing Torque from being used on Centos 7
+- Start node daemons at the end of instance initialization. The time spent for post-install script and node
+  initialization is not counted as part of node idletime anymore.
+- Fix issue which was causing an additional and invalid EBS mount point to be added in case of multiple EBS
+- Install Slurm libpmpi/libpmpi2 that is distributed in a separate package since Slurm 17
+
+
 2.3.1
 -----
 

amis/packer_alinux.json

@@ -210,6 +210,7 @@
       "pause_before": "2m",
       "json" : {
         "cfncluster" : {
+          "cfn_region": "{{user `region`}}",
           "nvidia" : {
             "enabled" : "{{user `nvidia_enabled`}}"
           },
@@ -246,7 +247,6 @@
     },
     {
       "type" : "shell",
-      "only": ["custom-alinux"],
       "inline" : [
         "sudo /usr/local/sbin/ami_cleanup.sh"
       ]

amis/packer_centos6.json

@@ -219,6 +219,7 @@
       "pause_before": "2m",
       "json" : {
         "cfncluster" : {
+          "cfn_region": "{{user `region`}}",
           "nvidia" : {
             "enabled" : "{{user `nvidia_enabled`}}"
           },
@@ -251,7 +252,7 @@
       "inline" : [
         "region=\"{{user `region`}}\"",
         "bucket=\"s3.amazonaws.com\"",
-        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn\"",
+        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster\"",
         "curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz",
         "sudo pip install /tmp/aws-cfn-bootstrap-latest.tar.gz"
       ]
@@ -265,7 +266,6 @@
     },
     {
       "type" : "shell",
-      "only": ["custom-centos6"],
       "inline" : [
         "sudo /usr/local/sbin/ami_cleanup.sh"
       ]

amis/packer_centos7.json

@@ -224,6 +224,7 @@
       "pause_before": "2m",
       "json" : {
         "cfncluster" : {
+          "cfn_region": "{{user `region`}}",
           "nvidia" : {
             "enabled" : "{{user `nvidia_enabled`}}"
           },
@@ -256,7 +257,7 @@
       "inline" : [
         "region=\"{{user `region`}}\"",
         "bucket=\"s3.amazonaws.com\"",
-        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn\"",
+        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster\"",
         "curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz",
         "which pip2",
         "if [ $? -eq 0 ]; then sudo pip2 install /tmp/aws-cfn-bootstrap-latest.tar.gz; else sudo pip install /tmp/aws-cfn-bootstrap-latest.tar.gz; fi"
@@ -277,7 +278,6 @@
     },
     {
       "type" : "shell",
-      "only": ["custom-centos7"],
       "inline" : [
         "sudo /usr/local/sbin/ami_cleanup.sh"
       ]

amis/packer_ubuntu1404.json

@@ -224,6 +224,7 @@
       "pause_before": "2m",
       "json" : {
         "cfncluster" : {
+          "cfn_region": "{{user `region`}}",
           "nvidia" : {
             "enabled" : "{{user `nvidia_enabled`}}"
           },
@@ -257,7 +258,7 @@
       "inline" : [
         "region=\"{{user `region`}}\"",
         "bucket=\"s3.amazonaws.com\"",
-        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn\"",
+        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster\"",
         "curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz",
         "sudo pip install /tmp/aws-cfn-bootstrap-latest.tar.gz"
       ]
@@ -271,7 +272,6 @@
     },
     {
       "type" : "shell",
-      "only": ["custom-ubuntu1404"],
       "inline" : [
         "sudo /usr/local/sbin/ami_cleanup.sh"
       ]

amis/packer_ubuntu1604.json

@@ -227,6 +227,7 @@
       "pause_before": "2m",
       "json" : {
         "cfncluster" : {
+          "cfn_region": "{{user `region`}}",
           "nvidia" : {
             "enabled" : "{{user `nvidia_enabled`}}"
           },
@@ -260,7 +261,7 @@
       "inline" : [
         "region=\"{{user `region`}}\"",
         "bucket=\"s3.amazonaws.com\"",
-        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn\"",
+        "[[ ${region} =~ ^cn- ]] && bucket=\"s3.cn-north-1.amazonaws.com.cn/cn-north-1-aws-parallelcluster\"",
         "curl --retry 3 -L -o /tmp/aws-cfn-bootstrap-latest.tar.gz https://${bucket}/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz",
         "sudo pip install /tmp/aws-cfn-bootstrap-latest.tar.gz"
       ]
@@ -274,7 +275,6 @@
     },
     {
       "type" : "shell",
-      "only": ["custom-ubuntu1604"],
       "inline" : [
         "sudo /usr/local/sbin/ami_cleanup.sh"
       ]

amis/packer_variables.json

@@ -1,6 +1,6 @@
 {
-  "parallelcluster_version": "2.3.1",
-  "parallelcluster_cookbook_version": "2.3.1",
+  "parallelcluster_version": "2.4.0",
+  "parallelcluster_cookbook_version": "2.4.0",
   "chef_version": "14.2.0",
   "ridley_version": "5.1.1",
   "berkshelf_version": "7.0.4"

attributes/default.rb

@@ -19,9 +19,9 @@
 default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts"
 default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses"
 # Python packages
-default['cfncluster']['cfncluster-version'] = '2.3.1'
-default['cfncluster']['cfncluster-node-version'] = '2.3.1'
-default['cfncluster']['cfncluster-supervisor-version'] = '3.3.1'
+default['cfncluster']['cfncluster-version'] = '2.4.0'
+default['cfncluster']['cfncluster-node-version'] = '2.4.0'
+default['cfncluster']['supervisor-version'] = '3.4.0'
 # URLs to software packages used during install recipes
 # Gridengine software
 default['cfncluster']['sge']['version'] = '8.1.9'
@@ -45,6 +45,8 @@
 default['cfncluster']['nvidia']['enabled'] = 'no'
 default['cfncluster']['nvidia']['driver_url'] = 'http://download.nvidia.com/XFree86/Linux-x86_64/418.56/NVIDIA-Linux-x86_64-418.56.run'
 default['cfncluster']['nvidia']['cuda_url'] = 'https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux'
+# EFA
+default['cfncluster']['efa']['installer_url'] = 'https://s3-us-west-2.amazonaws.com/aws-efa-installer/aws-efa-installer-latest.tar.gz'
 
 # Reboot after default_pre recipe
 default['cfncluster']['default_pre_reboot'] = 'true'
@@ -83,7 +85,7 @@
     if node['platform_version'].to_i >= 7
       default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
                                                   libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool
-                                                  httpd boost-devel redhat-lsb mlocate lvm2 mpich-devel openmpi-devel R atlas-devel
+                                                  httpd boost-devel redhat-lsb mlocate lvm2 mpich-devel R atlas-devel
                                                   blas-devel fftw-devel libffi-devel openssl-devel dkms mariadb-devel libedit-devel
                                                   libical-devel postgresql-devel postgresql-server sendmail libxml2-devel libglvnd-devel mdadm]
       if node['platform_version'].split('.')[1] == '6'
@@ -105,7 +107,7 @@
   when 'amazon'
     default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
                                                 libXmu-devel hwloc-devel db4-devel tcl-devel automake autoconf pyparted libtool
-                                                httpd boost-devel redhat-lsb mlocate mpich-devel openmpi-devel R atlas-devel fftw-devel
+                                                httpd boost-devel redhat-lsb mlocate mpich-devel R atlas-devel fftw-devel
                                                 libffi-devel openssl-devel dkms mysql-devel libedit-devel postgresql-devel postgresql-server
                                                 sendmail cmake byacc libglvnd-devel mdadm]
   end
@@ -123,8 +125,11 @@
   default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh libssl-dev ncurses-dev libpam-dev net-tools libhwloc-dev dkms
                                               tcl-dev automake autoconf python-parted libtool librrd-dev libapr1-dev libconfuse-dev
                                               apache2 libboost-dev libdb-dev tcsh libssl-dev libncurses5-dev libpam0g-dev libxt-dev
-                                              libmotif-dev libxmu-dev libxft-dev libhwloc-dev man-db lvm2 libmpich-dev libopenmpi-dev
+                                              libmotif-dev libxmu-dev libxft-dev libhwloc-dev man-db lvm2 libmpich-dev
                                               r-base libatlas-dev libblas-dev libfftw3-dev libffi-dev libssl-dev libxml2-dev mdadm]
+  if node['platform_version'] == '14.04'
+    default['cfncluster']['base_packages'].push('libopenmpi-dev')
+  end
   default['cfncluster']['kernel_generic_pkg'] = "linux-generic"
   default['cfncluster']['kernel_extra_pkg'] = "linux-image-extra-#{node['kernel']['release']}"
   default['cfncluster']['ganglia']['apache_user'] = 'www-data'
@@ -166,7 +171,6 @@
 default['cfncluster']['cfn_shared_dir'] = '/shared'
 default['cfncluster']['cfn_efs_shared_dir'] = 'NONE'
 default['cfncluster']['cfn_efs'] = nil
-default['cfncluster']['cfn_node_type'] = nil
 default['cfncluster']['cfn_master'] = nil
 default['cfncluster']['cfn_cluster_user'] = 'ec2-user'
 default['cfncluster']['cfn_fsx_options'] = 'NONE'

files/default/ami_cleanup.sh

@@ -5,5 +5,13 @@ rm -rf /var/lib/cloud/instances/*
 rm -f /var/lib/cloud/instance
 rm -rf /etc/ssh/ssh_host_*
 rm -f /etc/udev/rules.d/70-persistent-net.rules
+grep -l "Created by cloud-init on instance boot automatically" /etc/sysconfig/network-scripts/ifcfg-* | xargs rm -f
+
+# https://bugs.centos.org/view.php?id=13836#c33128
+source /etc/os-release
+if [ "${ID}${VERSION_ID}" == "centos7" ]; then
+    rm -f /etc/sysconfig/network-scripts/ifcfg-eth0
+fi
+
 find /var/log -type f -exec /bin/rm -v {} \;
-touch /var/log/lastlog
+touch /var/log/lastlog

files/default/compute_ready

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 . /etc/parallelcluster/cfnconfig
 

files/default/setup-ephemeral-drives.sh

@@ -21,6 +21,13 @@ function error_exit () {
   exit 1
 }
 
+function exec_command() {
+  _command_output=$($@ 2>&1)
+  _exit_code=$?
+
+  # Do not set RC=1 if error says that changes have been written but a reboot is required to inform the kernel
+  [[ $_exit_code -ne 0 && $(echo "${_command_output}" | grep -i "you should reboot now") ]] && RC=1
+}
 
 # LVM stripe, format, mount ephemeral drives
 function setup_ephemeral_drives () {
@@ -48,11 +55,11 @@ function setup_ephemeral_drives () {
     for d in $DEVS; do
       d=/dev/${d}
       dd if=/dev/zero of=${d} bs=32k count=1 || RC=1
-      parted -s ${d} mklabel gpt || RC=1
-      parted -s ${d} || RC=1
-      parted -s -a optimal ${d} mkpart primary 1MB 100% || RC=1
-      partprobe
-      parted -s ${d} set 1 lvm on || RC=1
+      exec_command "parted -s ${d} mklabel gpt"
+      exec_command "parted -s ${d}"
+      exec_command "parted -s -a optimal ${d} mkpart primary 1MB 100%"
+      partprobe ${d}
+      exec_command "parted -s ${d} set 1 lvm on"
       if [ $IS_NVME -eq 1 ]; then
         PARTITIONS="${d}p1 $PARTITIONS"
       else