From 41e189f10b97006498a5d8d3311c2cae21c81663 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 7 Oct 2025 17:01:14 -0400 Subject: [PATCH 1/3] [Isolated] Remove use of new_resource as this will be empty without a default value set --- .../resources/efs/efs_redhat8.rb | 2 +- .../resources/efs/partial/_common.rb | 7 ++----- .../resources/efs/partial/_install_from_tar.rb | 8 ++++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index abe333a933..6af950560c 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -42,7 +42,7 @@ def prerequisites action :install_efs_utils do package_name = "amazon-efs-utils" - package_version = new_resource.efs_utils_version + package_version = _efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" if aws_region.start_with?("us-iso") diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb index 618e91d3fb..612edabe20 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_common.rb @@ -2,15 +2,12 @@ default_action :install_utils -property :efs_utils_version, String -property :efs_utils_checksum, String - def _efs_utils_version - efs_utils_version || node['cluster']['efs']['version'] + node['cluster']['efs']['version'] end def _efs_utils_checksum - efs_utils_checksum || node['cluster']['efs']['sha256'] + node['cluster']['efs']['sha256'] end def already_installed?(package_name, expected_version) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index c3e320c906..2337b33917 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -15,7 +15,7 @@ package_name = "amazon-efs-utils" action :install_utils do - package_version = new_resource.efs_utils_version + package_version = _efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" efs_utils_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/v#{package_version}.tar.gz" @@ -46,7 +46,7 @@ mode '0644' retries 3 retry_delay 5 - checksum new_resource.efs_utils_checksum + checksum _efs_utils_checksum action :create_if_missing end @@ -61,9 +61,9 @@ end action :install_efs_utils do - package_version = new_resource.efs_utils_version + package_version = _efs_utils_version efs_utils_tarball = "#{node['cluster']['sources_dir']}/efs-utils-#{package_version}.tar.gz" - + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir'] code install_script_code(efs_utils_tarball, package_name, package_version) From ec644a64fd5d2c1ef6c3651fd5237ec4e1461b1a Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 7 Oct 2025 19:14:18 -0400 Subject: [PATCH 2/3] [Isolated] Update unit tests --- .../spec/unit/resources/efs_spec.rb | 53 ++++++++++++++----- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb index 738999a3eb..83f06bdf8b 100644 --- a/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb +++ b/cookbooks/aws-parallelcluster-environment/spec/unit/resources/efs_spec.rb @@ -1,11 +1,9 @@ require 'spec_helper' class ConvergeEfs - def self.install_utils(chef_run, efs_utils_version:, tarball_checksum:) + def self.install_utils(chef_run) chef_run.converge_dsl('aws-parallelcluster-environment') do efs 'install_utils' do - efs_utils_checksum tarball_checksum - efs_utils_version efs_utils_version action :install_utils end end @@ -28,14 +26,17 @@ def mock_already_installed(package, expected_version, installed) context "on amazon2" do cached(:efs_utils_version) { '1.2.3' } cached(:tarball_checksum) { 'tarball_checksum' } - let(:chef_run) do - runner(platform: 'amazon', version: '2', step_into: ['efs']) - end context "when same version of amazon-efs-utils already installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', efs_utils_version) - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'does not install amazon-efs-utils' do @@ -44,9 +45,13 @@ def mock_already_installed(package, expected_version, installed) end context "when newer version of amazon-efs-utils already installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do mock_get_package_version('amazon-efs-utils', '1.3.2') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'does not install amazon-efs-utils' do @@ -55,9 +60,15 @@ def mock_already_installed(package, expected_version, installed) end context "when amazon-efs-utils not installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', '') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'installs amazon-efs-utils' do @@ -67,9 +78,15 @@ def mock_already_installed(package, expected_version, installed) end context "when older version of amazon-efs-utils installed" do + cached(:chef_run) do + runner(platform: 'amazon', version: '2', step_into: ['efs']) + end + cached(:node) { chef_run.node } before do + node.override['cluster']['efs']['version'] = efs_utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum mock_get_package_version('amazon-efs-utils', '1.1.4') - ConvergeEfs.install_utils(chef_run, efs_utils_version: efs_utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(chef_run) end it 'installs amazon-efs-utils' do @@ -105,8 +122,10 @@ def mock_already_installed(package, expected_version, installed) node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir node.override['cluster']['region'] = aws_region + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end cached(:node) { chef_run.node } @@ -140,8 +159,10 @@ def mock_already_installed(package, expected_version, installed) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end cached(:node) { chef_run.node } @@ -190,9 +211,11 @@ def mock_already_installed(package, expected_version, installed) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path node.override['cluster']['sources_dir'] = source_dir + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum node.override['cluster']['region'] = aws_region end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end it 'creates sources dir' do @@ -230,9 +253,11 @@ def mock_already_installed(package, expected_version, installed) mock_already_installed('amazon-efs-utils', utils_version, true) runner = runner(platform: platform, version: version, step_into: ['efs']) do |node| node.override['cluster']['efs_utils']['tarball_path'] = tarball_path + node.override['cluster']['efs']['version'] = utils_version + node.override['cluster']['efs']['sha256'] = tarball_checksum node.override['cluster']['sources_dir'] = source_dir end - ConvergeEfs.install_utils(runner, efs_utils_version: utils_version, tarball_checksum: tarball_checksum) + ConvergeEfs.install_utils(runner) end it 'does not download tarball' do From 833985d6efbdb0446bf7974954e9ccfb96685de9 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 8 Oct 2025 14:33:51 -0400 Subject: [PATCH 3/3] [Bug] Install cookbook-dependencies in all regions --- .../recipes/install/cookbook_virtualenv.rb | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index 8717b06a69..4a0c149f10 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -33,24 +33,22 @@ not_if { ::File.exist?("#{cookbook_virtualenv_path}/bin/activate") } end -if aws_region.start_with?("us-iso") - remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do - source pypi_s3_uri - mode '0644' - retries 3 - retry_delay 5 - action :create_if_missing - end +remote_file "#{node['cluster']['base_dir']}/cookbook-dependencies.tgz" do + source pypi_s3_uri + mode '0644' + retries 3 + retry_delay 5 + action :create_if_missing +end - bash 'pip install' do - user 'root' - group 'root' - cwd "#{node['cluster']['base_dir']}" - code <<-REQ - set -e - tar xzf cookbook-dependencies.tgz - cd #{dependency_package_name} - #{virtualenv_path}/bin/pip install * -f ./ --no-index - REQ - end +bash 'pip install' do + user 'root' + group 'root' + cwd "#{node['cluster']['base_dir']}" + code <<-REQ + set -e + tar xzf cookbook-dependencies.tgz + cd #{dependency_package_name} + #{virtualenv_path}/bin/pip install * -f ./ --no-index + REQ end