Skip to content

Commit e43fc9d

Browse files
metax666metax666
authored
[Metax] add private CI (#2139)
1 parent fefd4de commit e43fc9d

File tree

10 files changed

+833
-187
lines changed

10 files changed

+833
-187
lines changed

‎.github/workflows/metax_work.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ jobs:
7878

7979
run: |
8080
cd backends/metax_gpu/tests
81-
bash run_test.sh -j 16
81+
bash run_test.sh -j 8
8282
8383
- name: push whl
8484
env:

‎.github/workflows/metax_work_private.yaml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
name: paddle metax gpu private test
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
types: [opened, synchronize]
7+
branches: [develop, release/**]
8+
schedule:
9+
- cron: "0 15 * * *"
10+
permissions: read-all
11+
12+
defaults:
13+
run:
14+
shell: bash
15+
16+
jobs:
17+
metax-gpu-test:
18+
runs-on: paddle-metax-runner-set
19+
# runs-on: debug-paddle-runner-set
20+
steps:
21+
- name: Checkout repository
22+
run: |
23+
git config --global user.name "GitHub Actions"
24+
git config --global user.email "[email protected]"
25+
26+
git clone \
27+
--reference-if-able /home/runner/PaddleCustomDevice \
28+
--depth=1 \
29+
--shallow-submodules \
30+
--jobs=8 \
31+
--branch ${{ github.base_ref || github.ref_name}} \
32+
--recurse-submodules \
33+
https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git .
34+
35+
if [ "${{ github.event_name }}" == "pull_request" ]; then
36+
git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/head
37+
git checkout pull/${{ github.event.pull_request.number }}/head
38+
39+
40+
41+
42+
paddle_branch=${{ github.base_ref || github.ref_name}}
43+
echo $paddle_branch
44+
# sleep 10000
45+
change_numbers=$(git diff --name-only remotes/origin/${paddle_branch} | wc -l)
46+
echo $change_numbers
47+
48+
49+
change_backend=$(git diff --name-only remotes/origin/${paddle_branch} | grep -c "backends/" || true)
50+
echo $change_backend
51+
change_metax_only=$(git diff --name-only remotes/origin/${paddle_branch} | grep -c "backends/metax_gpu" || true)
52+
echo $change_metax_only
53+
54+
# change_backend=$(git diff --name-only remotes/origin/${paddle_branch} | grep "backends/"| wc -l)
55+
# echo $change_backend
56+
# change_metax_only=$(git diff --name-only remotes/origin/${paddle_branch} | grep "backends/metax_gpu"| wc -l)
57+
# echo $change_metax_only
58+
59+
git diff --name-only remotes/origin/${paddle_branch}
60+
61+
if [ $change_numbers -ne $change_backend ]; then
62+
echo "Common file changed, continue to run metax FULL CI test ..."
63+
elif [ $paddle_branch -eq 0 ] ; then
64+
echo "NO metax backend changes found, skip metax FULL CI ....."
65+
exit 0
66+
fi
67+
68+
69+
# git submodule update --init --recursive
70+
fi
71+
72+
73+
- name: compile
74+
run: |
75+
# sleep 10000
76+
cd backends/metax_gpu
77+
bash build_private_CI.sh
78+
79+
- name: run test
80+
81+
run: |
82+
cd backends/metax_gpu/tests
83+
bash run_test.sh -j 8
84+
85+
- name: push whl
86+
env:
87+
PR_ID: ${{ github.event.pull_request.number }}
88+
COMMIT_ID: ${{ github.event.pull_request.head.sha }}
89+
run: |
90+
pip install bce-python-sdk==0.8.74
91+
export AK=paddle
92+
export SK=paddle
93+
if [ ! -f "BosClient.py}" ]; then
94+
wget -q --no-proxy https://xly-devops.bj.bcebos.com/home/bos_retry.tar.gz --no-check-certificate
95+
tar xf bos_retry.tar.gz
96+
fi
97+
cp backends/metax_gpu/build/dist/paddle_metax_gpu*.whl .
98+
python BosClient.py paddle_metax_gpu*.whl paddle-github-action/PaddleCustomDevice/metax_gpu/${PR_ID}/${COMMIT_ID}

‎backends/metax_gpu/build_private_CI.sh‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# 2024 - Modified by MetaX Integrated Circuits (Shanghai) Co., Ltd. All Rights Reserved.
2+
#!/bin/bash
3+
4+
# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
set -e
19+
# uninstall paddle
20+
pip uninstall paddlepaddle -y
21+
22+
23+
#!/bin/bash
24+
25+
# update_paddle_dev.sh
26+
27+
chown -R $USER:$USER ../../Paddle/
28+
chown -R $USER:$USER ../../../PaddleCustomDevice/
29+
# Step 1: 撤销所有本地修改（已跟踪的文件，不包括新文件）
30+
cd ../../Paddle/
31+
echo "🔄 正在撤销所有本地修改（git checkout .）..."
32+
git checkout develop
33+
git checkout .
34+
35+
# Step 2: 拉取远程最新的 dev (通常是 develop) 分支代码
36+
echo "🌐 正在拉取远程最新的 dev (develop) 分支代码..."
37+
38+
39+
# 拉取 develop 分支的最新代码（与远程同步）
40+
git pull origin develop
41+
42+
echo "🔗 当前分支: $(git branch --show-current)"
43+
echo "📌 最新 commit hash (短): $(git rev-parse --short HEAD)"
44+
echo "📌 最新 commit 信息:"
45+
git log -1 --oneline
46+
47+
# 提示完成
48+
echo "✅ 脚本执行完毕！"
49+
echo "📌 已撤销本地修改，并更新到 Paddle 最新的 develop (dev) 分支代码。"
50+
51+
52+
pip install parameterized safetensors==0.6.2 -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple some-package
53+
# install paddle
54+
55+
python -m pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/
56+
57+
58+
# unset http_proxy https_proxy
59+
cd -
60+
# apply patch
61+
bash change_patch.sh
62+
63+
export MACA_PATH=/opt/maca
64+
export CUDA_PATH=/workspace/cuda-11.7/
65+
export PATH=${CUDA_PATH}/bin:${PATH}
66+
export CUCC_PATH=${MACA_PATH}/tools/cu-bridge
67+
export PATH=${PATH}:${CUCC_PATH}/tools:${CUCC_PATH}/bin
68+
export PATH=${MACA_PATH}/bin:${PATH}
69+
export LD_LIBRARY_PATH=${MACA_PATH}/lib:${MACA_PATH}/mxgpu_llvm/lib:${LD_LIBRARY_PATH}
70+
export PADDLE_VERSION=3.3.0
71+
72+
if [ ! -d build ]; then
73+
echo "build directory not found, creating..."
74+
mkdir build
75+
fi
76+
77+
echo "make_maca"
78+
cd build
79+
cmake_maca .. -DCMAKE_BUILD_TYPE=Release -DPython3_EXECUTABLE=$(which python3) -DWITH_GPU=ON
80+
make_maca -j60
81+
82+
echo "install whl"
83+
pip install dist/paddle_metax_gpu*.whl --force-reinstall
84+
cd ..
85+
echo "Done!"
86+
87+
cd build/dist/
88+
ossutil ls oss://opensource-ci/paddle/
89+
ossutil cat oss://opensource-ci/paddle/test1
90+
ossutil cp ./paddle_metax_gpu-*.whl oss://opensource-ci/paddle/test1/
91+
cd -

‎backends/metax_gpu/kernels/cuda_kernels/uniform_kernel_register.cu‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,8 @@ PD_CUSTOM_KERNEL_REGISTER(uniform,
2121
phi::UniformKernel,
2222
float,
2323
double,
24-
phi::dtype::float16,
25-
phi::dtype::bfloat16) {}
24+
phi::float16,
25+
phi::bfloat16,
26+
phi::float8_e4m3fn,
27+
phi::complex64,
28+
phi::complex128) {}

‎backends/metax_gpu/patch/paddle.patch‎

Lines changed: 10 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ index bff0f2bf70..9376b5781f 100644
4848
#include "paddle/phi/core/platform/device/gpu/gpu_info.h"
4949
#include "paddle/phi/core/platform/profiler/utils.h"
5050
diff --git a/paddle/phi/backends/dynload/cublas.h b/paddle/phi/backends/dynload/cublas.h
51-
index 62beb53cfe..0b0ac09fc0 100644
51+
index bda9cbe17e..c73eba9c8a 100644
5252
--- a/paddle/phi/backends/dynload/cublas.h
5353
+++ b/paddle/phi/backends/dynload/cublas.h
5454
@@ -49,7 +49,12 @@ extern void *cublas_dso_handle;
@@ -98,107 +98,21 @@ index 8b2e08c777..ca926df151 100644
9898
#define CUBLASLT_BLAS_ROUTINE_EACH(__macro) \
9999
__macro(cublasLtCreate); \
100100
diff --git a/paddle/phi/backends/dynload/cudnn.h b/paddle/phi/backends/dynload/cudnn.h
101-
index c0080f0a5e..458ca3e2e8 100644
101+
index a943bbed9a..af931490e3 100644
102102
--- a/paddle/phi/backends/dynload/cudnn.h
103103
+++ b/paddle/phi/backends/dynload/cudnn.h
104-
@@ -38,7 +38,9 @@ extern void EnforceCUDNNLoaded(const char* fn_name);
104+
@@ -38,7 +38,10 @@ extern void EnforceCUDNNLoaded(const char* fn_name);
105105
cudnn_dso_handle = phi::dynload::GetCUDNNDsoHandle(); \
106106
}); \
107107
EnforceCUDNNLoaded(#__name); \
108108
- static void* p_##__name = dlsym(cudnn_dso_handle, #__name); \
109109
+ std::string replaced_name = #__name; \
110-
+ replaced_name = replaced_name.replace(0,2,"mc"); \
111-
+ static void* p_##__name = dlsym(cudnn_dso_handle, replaced_name.c_str()); \
110+
+ replaced_name = replaced_name.replace(0, 2, "mc"); \
111+
+ static void* p_##__name = \
112+
+ dlsym(cudnn_dso_handle, replaced_name.c_str()); \
112113
return reinterpret_cast<cudnn_func>(p_##__name)(args...); \
113114
} \
114115
}; \
115-
@@ -49,7 +51,6 @@ extern void EnforceCUDNNLoaded(const char* fn_name);
116-
* different cudnn version has different interfaces
117-
**/
118-
#define CUDNN_DNN_ROUTINE_EACH(__macro) \
119-
- __macro(cudnnSetCallback); \
120-
__macro(cudnnSetTensor4dDescriptor); \
121-
__macro(cudnnSetTensor4dDescriptorEx); \
122-
__macro(cudnnSetTensorNdDescriptor); \
123-
@@ -104,6 +105,13 @@ extern void EnforceCUDNNLoaded(const char* fn_name);
124-
__macro(cudnnSetDropoutDescriptor); \
125-
__macro(cudnnRestoreDropoutDescriptor); \
126-
__macro(cudnnCreateRNNDescriptor); \
127-
+ __macro(cudnnGetRNNParamsSize); \
128-
+ __macro(cudnnGetRNNWorkspaceSize); \
129-
+ __macro(cudnnGetRNNTrainingReserveSize); \
130-
+ __macro(cudnnRNNForwardTraining); \
131-
+ __macro(cudnnRNNBackwardData); \
132-
+ __macro(cudnnRNNBackwardWeights); \
133-
+ __macro(cudnnRNNForwardInference); \
134-
__macro(cudnnDestroyDropoutDescriptor); \
135-
__macro(cudnnDestroyRNNDescriptor); \
136-
__macro(cudnnSetTensorNdDescriptorEx); \
137-
@@ -118,7 +126,8 @@ extern void EnforceCUDNNLoaded(const char* fn_name);
138-
__macro(cudnnCreateActivationDescriptor); \
139-
__macro(cudnnSetActivationDescriptor); \
140-
__macro(cudnnGetActivationDescriptor); \
141-
- __macro(cudnnDestroyActivationDescriptor);
142-
+ __macro(cudnnDestroyActivationDescriptor); \
143-
+ __macro(cudnnSetRNNDescriptor_v6);
144-
CUDNN_DNN_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
145-
146-
#if CUDNN_VERSION >= 7000 && CUDNN_VERSION < 8000
147-
@@ -152,7 +161,12 @@ CUDNN_DNN_ROUTINE_EACH_R7(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
148-
#define CUDNN_DNN_ROUTINE_EACH_AFTER_TWO_R7(__macro) \
149-
__macro(cudnnCreateRNNDataDescriptor); \
150-
__macro(cudnnDestroyRNNDataDescriptor); \
151-
- __macro(cudnnSetRNNDataDescriptor);
152-
+ __macro(cudnnSetRNNDataDescriptor); \
153-
+ __macro(cudnnSetRNNPaddingMode); \
154-
+ __macro(cudnnRNNForwardTrainingEx); \
155-
+ __macro(cudnnRNNBackwardDataEx); \
156-
+ __macro(cudnnRNNBackwardWeightsEx); \
157-
+ __macro(cudnnRNNForwardInferenceEx);
158-
CUDNN_DNN_ROUTINE_EACH_AFTER_TWO_R7(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
159-
#endif
160-
161-
@@ -195,40 +209,6 @@ CUDNN_DNN_ROUTINE_EACH_R8(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
162-
CUDNN_DNN_ROUTINE_EACH_FRONTEND(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
163-
#endif
164-
165-
-#if CUDNN_VERSION < 90000
166-
-#define CUDNN_DNN_ROUTINE_EACH_REMOVED_IN_E9(__macro) \
167-
- __macro(cudnnGetRNNParamsSize); \
168-
- __macro(cudnnGetRNNWorkspaceSize); \
169-
- __macro(cudnnGetRNNTrainingReserveSize); \
170-
- __macro(cudnnSetRNNDescriptor_v6); \
171-
- __macro(cudnnRNNForwardInference); \
172-
- __macro(cudnnRNNForwardTraining); \
173-
- __macro(cudnnRNNBackwardData); \
174-
- __macro(cudnnRNNBackwardWeights);
175-
-CUDNN_DNN_ROUTINE_EACH_REMOVED_IN_E9(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
176-
-#endif
177-
-
178-
-#if CUDNN_VERSION < 90000 && CUDNN_VERSION >= 7201
179-
-#define CUDNN_DNN_ROUTINE_EACH_AFTER_TWO_R7_REMOVED_IN_E9(__macro) \
180-
- __macro(cudnnSetRNNPaddingMode); \
181-
- __macro(cudnnRNNForwardInferenceEx); \
182-
- __macro(cudnnRNNForwardTrainingEx); \
183-
- __macro(cudnnRNNBackwardDataEx); \
184-
- __macro(cudnnRNNBackwardWeightsEx);
185-
-CUDNN_DNN_ROUTINE_EACH_AFTER_TWO_R7_REMOVED_IN_E9(
186-
- DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
187-
-#endif
188-
-
189-
-#if CUDNN_VERSION >= 90000
190-
-#define CUDNN_DNN_ROUTINE_EACH_R9(__macro) \
191-
- __macro(cudnnGetLastErrorString); \
192-
- __macro(cudnnGetRNNWeightSpaceSize); \
193-
- __macro(cudnnGetRNNTempSpaceSizes); \
194-
- __macro(cudnnRNNForward); \
195-
- __macro(cudnnRNNBackwardData_v8); \
196-
- __macro(cudnnRNNBackwardWeights_v8);
197-
-CUDNN_DNN_ROUTINE_EACH_R9(DECLARE_DYNAMIC_LOAD_CUDNN_WRAP)
198-
-#endif
199-
} // namespace dynload
200-
} // namespace phi
201-
202116
diff --git a/paddle/phi/backends/dynload/cufft.h b/paddle/phi/backends/dynload/cufft.h
203117
index 1547909d92..ef20838434 100644
204118
--- a/paddle/phi/backends/dynload/cufft.h
@@ -247,7 +161,7 @@ index 59e92955c9..d2f8c2da15 100644
247161
+#endif // PADDLE_WITH_CUPTI
248162
\ No newline at end of file
249163
diff --git a/paddle/phi/backends/dynload/cusolver.h b/paddle/phi/backends/dynload/cusolver.h
250-
index 86651fc8f1..7c9b122a17 100644
164+
index 57e09bb6e4..87fb5b1797 100644
251165
--- a/paddle/phi/backends/dynload/cusolver.h
252166
+++ b/paddle/phi/backends/dynload/cusolver.h
253167
@@ -34,7 +34,9 @@ extern void *cusolver_dso_handle;
@@ -262,7 +176,7 @@ index 86651fc8f1..7c9b122a17 100644
262176
} \
263177
}; \
264178
diff --git a/paddle/phi/backends/dynload/cusparse.h b/paddle/phi/backends/dynload/cusparse.h
265-
index 8ec3cf2792..6f5460df00 100644
179+
index e8cb0ac643..e8e7596d44 100644
266180
--- a/paddle/phi/backends/dynload/cusparse.h
267181
+++ b/paddle/phi/backends/dynload/cusparse.h
268182
@@ -34,7 +34,9 @@ extern void *cusparse_dso_handle;
@@ -277,7 +191,7 @@ index 8ec3cf2792..6f5460df00 100644
277191
} \
278192
}; \
279193
diff --git a/paddle/phi/backends/dynload/dynamic_loader.cc b/paddle/phi/backends/dynload/dynamic_loader.cc
280-
index 859f696896..87b5100a1b 100644
194+
index c74ae9592e..f6dc68917c 100644
281195
--- a/paddle/phi/backends/dynload/dynamic_loader.cc
282196
+++ b/paddle/phi/backends/dynload/dynamic_loader.cc
283197
@@ -18,7 +18,6 @@ limitations under the License. */
@@ -755,7 +669,7 @@ index 4eae698648..5c047723ea 100644
755669
return block_dim >= kMaxBlockDim ? kMaxBlockDim : lwarpSize;
756670
}
757671
diff --git a/paddle/phi/kernels/funcs/math_cuda_utils.h b/paddle/phi/kernels/funcs/math_cuda_utils.h
758-
index e5361b836e..5ad238df08 100644
672+
index dff1033db4..0098123818 100644
759673
--- a/paddle/phi/kernels/funcs/math_cuda_utils.h
760674
+++ b/paddle/phi/kernels/funcs/math_cuda_utils.h
761675
@@ -175,12 +175,12 @@ struct KeyValuePair<half> {

0 commit comments

Comments
 (0)