[Refiled] IFU-main-2023-07-31

common/install_rocm.sh

@@ -11,7 +11,7 @@ ver() {
 }
 
 # Map ROCm version to AMDGPU version
-declare -A AMDGPU_VERSIONS=( ["5.0"]="21.50" ["5.1.1"]="22.10.1" ["5.2"]="22.20" )
+declare -A AMDGPU_VERSIONS=( ["5.0"]="21.50" ["5.1.1"]="22.10.1" ["5.1.3"]="22.10.3" ["5.2"]="22.20" )
 
 install_ubuntu() {
     apt-get update

common/install_rocm_miopen_rpm.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -ex
+
+# Create the tmp dir to extract into
+EXTRACTDIR_ROOT=/extract_miopen_rpms
+mkdir -p ${EXTRACTDIR_ROOT}
+echo "Creating temporary directory for rpm download..."
+
+MIOPEN_RPM_SOURCE_ARRAY=($MIOPEN_RPM_SOURCE)
+# Fail if rpm source is not available
+#if [[ ! wget -P ${EXTRACTDIR_ROOT} ${MIOPEN_RPM_SOURCE_ARRAY[0]} ]] || [[ ! wget -P ${EXTRACTDIR_ROOT} ${MIOPEN_RPM_SOURCE_ARRAY[1]} ]]; then
+if ! wget -P ${EXTRACTDIR_ROOT} ${MIOPEN_RPM_SOURCE_ARRAY[0]} || ! wget -P ${EXTRACTDIR_ROOT} ${MIOPEN_RPM_SOURCE_ARRAY[1]}; then
+  echo 'ERROR: Failed to download MIOpen package.'
+  exit 1
+fi
+echo "MIOpen package download complete..."
+
+# Extract rpm in EXTRACT_DIR
+cd ${EXTRACTDIR_ROOT}
+miopen_rpms=$(ls *.rpm)
+for miopen_rpm in ${miopen_rpms}; do
+  rpm2cpio ${miopen_rpm} | cpio -idmv
+done
+
+# Copy libMIOpen.so, headers and db files over existing
+files="lib/ include/ share/miopen/db/"
+for file in $files; do
+  source="opt/rocm*/${file}/*"
+  dest="/opt/rocm/${file}"
+  # Use backslash to run cp in (non-aliased) non-interactive mode
+  \cp -dR $source $dest
+done
+echo "libMIOpen so file from RPM copied to existing MIOpen install..."
+
+# Clean up extracted dir
+rm -rf ${EXTRACTDIR_ROOT}
+echo "Removed temporary directory..."

manywheel/Dockerfile

@@ -158,6 +158,7 @@ ENV PATH=/usr/local/cuda/bin:$PATH
 FROM cpu_final as rocm_final
 ARG ROCM_VERSION=3.7
 ARG PYTORCH_ROCM_ARCH
+ARG PYTORCH_BRANCH
 ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
 # No need to install ROCm as base docker image should have full ROCm install
 #ADD ./common/install_rocm.sh install_rocm.sh
@@ -168,5 +169,6 @@ RUN bash ./install_rocm_drm.sh && rm install_rocm_drm.sh
 RUN ln -sf /usr/local/bin/cmake /usr/bin/cmake3
 ADD ./common/install_rocm_magma.sh install_rocm_magma.sh
 RUN bash ./install_rocm_magma.sh && rm install_rocm_magma.sh
-ADD ./common/install_miopen.sh install_miopen.sh
-RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh
+# Do not install miopen from source for ROCm fork until an env var is introduced to control the behavior
+#ADD ./common/install_miopen.sh install_miopen.sh
+#RUN bash ./install_miopen.sh ${ROCM_VERSION} && rm install_miopen.sh

manywheel/build_rocm.sh

@@ -14,6 +14,8 @@ export USE_STATIC_NCCL=1
 export ATEN_STATIC_CUDA=1
 export USE_CUDA_STATIC_LINK=1
 export INSTALL_TEST=0 # dont install test binaries into site-packages
+# Set RPATH instead of RUNPATH when using patchelf to avoid LD_LIBRARY_PATH override
+export FORCE_RPATH="--force-rpath"
 
 # Keep an array of cmake variables to add to
 if [[ -z "$CMAKE_ARGS" ]]; then
@@ -202,10 +204,17 @@ if [[ $ROCM_INT -ge 50500 ]]; then
 
     DEPS_AUX_SRCLIST+=(${MIOPEN_SHARE_FILES[@]/#/$MIOPEN_SHARE_SRC/})
     DEPS_AUX_DSTLIST+=(${MIOPEN_SHARE_FILES[@]/#/$MIOPEN_SHARE_DST/})
-elif [[ $ROCM_INT -ge 50600 ]]; then
+fi
+
+if [[ $ROCM_INT -ge 50600 ]]; then
     # RCCL library files
-    RCCL_SHARE_SRC=$ROCM_HOME/lib/msccl-algorithms
-    RCCL_SHARE_DST=lib/msccl-algorithms
+    if [[ $ROCM_INT -ge 50700 ]]; then
+        RCCL_SHARE_SRC=$ROCM_HOME/share/rccl/msccl-algorithms
+        RCCL_SHARE_DST=share/rccl/msccl-algorithms
+    else
+        RCCL_SHARE_SRC=$ROCM_HOME/lib/msccl-algorithms
+        RCCL_SHARE_DST=lib/msccl-algorithms
+    fi
     RCCL_SHARE_FILES=($(ls $RCCL_SHARE_SRC))
 
     DEPS_AUX_SRCLIST+=(${RCCL_SHARE_FILES[@]/#/$RCCL_SHARE_SRC/})