Skip to content

Commit a0028c1

Browse files
borsbors
committed
Auto merge of #136247 - marcoieni:free-more-space-ubuntu-24, r=<try>
[experiment] ci: free more space in ubuntu 24 free runners try-job: x86_64-gnu-debug try-job: armhf-gnu try-job: dist-powerpc64le-linux try-job: dist-ohos try-job: aarch64-gnu
2 parents 25a1657 + 883074b commit a0028c1

File tree

2 files changed

+194
-99
lines changed

2 files changed

+194
-99
lines changed

‎src/ci/github-actions/jobs.yml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,11 @@ runners:
55
env: { }
66

77
- &job-linux-4c
8-
os: ubuntu-22.04
8+
os: ubuntu-24.04
99
# Free some disk space to avoid running out of space during the build.
1010
free_disk: true
1111
<<: *base-job
1212

13-
# Large runner used mainly for its bigger disk capacity
14-
- &job-linux-4c-largedisk
15-
os: ubuntu-22.04-4core-16gb
16-
<<: *base-job
17-
1813
- &job-linux-8c
1914
os: ubuntu-22.04-8core-32gb
2015
<<: *base-job
@@ -50,7 +45,7 @@ runners:
5045
- &job-aarch64-linux
5146
# Free some disk space to avoid running out of space during the build.
5247
free_disk: true
53-
os: ubuntu-22.04-arm
48+
os: ubuntu-24.04-arm
5449

5550
- &job-aarch64-linux-8c
5651
os: ubuntu-22.04-arm64-8core-32gb
@@ -182,7 +177,7 @@ auto:
182177
<<: *job-linux-4c
183178

184179
- name: dist-powerpc64le-linux
185-
<<: *job-linux-4c-largedisk
180+
<<: *job-linux-4c
186181

187182
- name: dist-riscv64-linux
188183
<<: *job-linux-4c
@@ -295,7 +290,7 @@ auto:
295290
- name: x86_64-gnu-debug
296291
# This seems to be needed because a full stage 2 build + run-make tests
297292
# overwhelms the storage capacity of the standard 4c runner.
298-
<<: *job-linux-4c-largedisk
293+
<<: *job-linux-4c
299294

300295
- name: x86_64-gnu-distcheck
301296
<<: *job-linux-8c

‎src/ci/scripts/free-disk-space.sh‎

Lines changed: 190 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,74 @@
11
#!/bin/bash
2+
set -euo pipefail
23

3-
# Free disk space on Linux GitHub action runners
4+
# Free disk space on Linux GitHub action runners.
45
# Script inspired by https://github.com/jlumbroso/free-disk-space
56

6-
# print a line of the specified character
7+
# When updating to a new ubuntu version (e.g. from ubuntu-24.04):
8+
# - Check that there are no docker images preinstalled with `docker image ls`
9+
# - Check that there are no big packages preinstalled that we aren't using
10+
# - Check that all directores we are removing are still present (look at the warnings)
11+
12+
# Print a line of the specified character.
713
printSeparationLine() {
814
for ((i = 0; i < 80; i++)); do
915
printf "%s" "$1"
1016
done
1117
printf "\n"
1218
}
1319

14-
# compute available space
20+
# Compute available space.
1521
# REF: https://unix.stackexchange.com/a/42049/60849
1622
# REF: https://stackoverflow.com/a/450821/408734
17-
getAvailableSpace() { echo $(df -a | awk 'NR > 1 {avail+=$4} END {print avail}'); }
23+
getAvailableSpace() {
24+
df -a | awk 'NR > 1 {avail+=$4} END {print avail}'
25+
}
1826

19-
# make Kb human readable (assume the input is Kb)
27+
# Make Kb human readable (assume the input is Kb).
2028
# REF: https://unix.stackexchange.com/a/44087/60849
21-
formatByteCount() { echo $(numfmt --to=iec-i --suffix=B --padding=7 $1'000'); }
29+
formatByteCount() {
30+
numfmt --to=iec-i --suffix=B --padding=7 "$1"'000'
31+
}
32+
33+
# Check if the architecture is x86.
34+
isX86() {
35+
local arch
36+
arch=$(uname -m)
37+
if [ "$arch" = "x86_64" ]; then
38+
return 0
39+
else
40+
return 1
41+
fi
42+
}
43+
44+
# Execute a task, printing how much space was saved and how long it took to run the task
45+
execAndMeasure() {
46+
local task_name=${1}
47+
48+
local start
49+
start=$(date +%s)
50+
51+
local before
52+
before=$(getAvailableSpace)
53+
54+
# Run the task. Skip the first argument because it's the task name.
55+
"${@:2}"
2256

23-
# macro to output saved space
24-
printSavedSpace() {
25-
# Disk space before the operation
26-
local before=${1}
27-
local title=${2:-}
57+
local end
58+
end=$(date +%s)
2859

2960
local after
3061
after=$(getAvailableSpace)
62+
63+
# How much space was saved.
3164
local saved=$((after - before))
65+
# How long the task took.
66+
local seconds_taken=$((end - start))
3267

33-
echo ""
34-
printSeparationLine "*"
35-
if [ -n "${title}" ]; then
36-
echo "=> ${title}: Saved $(formatByteCount "$saved")"
37-
else
38-
echo "=> Saved $(formatByteCount "$saved")"
39-
fi
40-
printSeparationLine "*"
41-
echo ""
68+
echo "==> ${task_name}: Saved $(formatByteCount "$saved") in $seconds_taken seconds"
4269
}
4370

44-
# macro to print output of df with caption
71+
# Print output of df with caption. It shows information about disk space.
4572
printDF() {
4673
local caption=${1}
4774

@@ -52,91 +79,164 @@ printDF() {
5279
echo ""
5380
df -h
5481
printSeparationLine "="
82+
echo ""
5583
}
5684

57-
removeDir() {
58-
dir=${1}
59-
60-
local before
61-
before=$(getAvailableSpace)
62-
63-
sudo rm -rf "$dir" || true
85+
removeUnusedDirsAndFiles() {
86+
local to_remove=(
87+
"/etc/mysql"
88+
"/usr/local/aws-sam-cli"
89+
"/usr/local/doc/cmake"
90+
"/usr/local/julia"*
91+
"/usr/local/lib/android"
92+
"/usr/local/share/chromedriver-"*
93+
"/usr/local/share/chromium"
94+
"/usr/local/share/cmake-"*
95+
"/usr/local/share/edge_driver"
96+
"/usr/local/share/gecko_driver"
97+
"/usr/local/share/icons"
98+
"/usr/local/share/vim"
99+
"/usr/local/share/emacs"
100+
"/usr/local/share/powershell"
101+
"/usr/local/share/vcpkg"
102+
"/usr/share/apache-maven-"*
103+
"/usr/share/gradle-"*
104+
"/usr/share/java"
105+
"/usr/share/kotlinc"
106+
"/usr/share/miniconda"
107+
"/usr/share/php"
108+
"/usr/share/ri"
109+
"/usr/share/swift"
110+
111+
# binaries
112+
"/usr/local/bin/azcopy"
113+
"/usr/local/bin/bicep"
114+
"/usr/local/bin/ccmake"
115+
"/usr/local/bin/cmake-"*
116+
"/usr/local/bin/cmake"
117+
"/usr/local/bin/cpack"
118+
"/usr/local/bin/ctest"
119+
"/usr/local/bin/helm"
120+
"/usr/local/bin/kind"
121+
"/usr/local/bin/kustomize"
122+
"/usr/local/bin/minikube"
123+
"/usr/local/bin/packer"
124+
"/usr/local/bin/phpunit"
125+
"/usr/local/bin/pulumi-"*
126+
"/usr/local/bin/pulumi"
127+
"/usr/local/bin/stack"
128+
129+
# Haskell runtime
130+
"/usr/local/.ghcup"
131+
132+
# Azure
133+
"/opt/az"
134+
"/usr/share/az_"*
135+
136+
# Environemnt variable set by GitHub Actions
137+
"$AGENT_TOOLSDIRECTORY"
138+
)
139+
140+
local existing=()
141+
for element in "${to_remove[@]}"; do
142+
if [ ! -e "$element" ]; then
143+
echo "::warning::Directory or file $element does not exist, skipping."
144+
else
145+
existing+=("$element")
146+
fi
147+
done
64148

65-
printSavedSpace "$before" "$dir"
149+
execAndMeasure "Removed unused directories" sudo rm -rf "${existing[@]}"
66150
}
67151

68-
execAndMeasureSpaceChange() {
69-
local operation=${1} # Function to execute
70-
local title=${2}
71-
72-
local before
73-
before=$(getAvailableSpace)
74-
$operation
75-
76-
printSavedSpace "$before" "$title"
152+
removeNodeModules() {
153+
sudo npm uninstall -g \
154+
"@bazel/bazelisk" \
155+
"grunt" \
156+
"gulp" \
157+
"lerna" \
158+
"n" \
159+
"newman" \
160+
"parcel" \
161+
"typescript" \
162+
"webpack-cli" \
163+
"webpack" \
164+
"yarn"
77165
}
78166

79-
# Remove large packages
80-
# REF: https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
167+
# Remove unused packages.
81168
cleanPackages() {
82-
sudo apt-get -qq remove -y --fix-missing \
83-
'^aspnetcore-.*' \
84-
'^dotnet-.*' \
85-
'^llvm-.*' \
86-
'php.*' \
87-
'^mongodb-.*' \
88-
'^mysql-.*' \
89-
'azure-cli' \
90-
'google-chrome-stable' \
91-
'firefox' \
92-
'powershell' \
93-
'mono-devel' \
94-
'libgl1-mesa-dri' \
95-
'google-cloud-sdk' \
96-
'google-cloud-cli'
169+
local packages=(
170+
'.*-icon-theme$'
171+
'^aspnetcore-.*'
172+
'^dotnet-.*'
173+
'^java-*'
174+
'^libllvm.*'
175+
'^llvm-.*'
176+
'^mercurial.*'
177+
'^mysql-.*'
178+
'^vim.*'
179+
'^fonts-.*'
180+
'azure-cli'
181+
'buildah'
182+
'cpp-13'
183+
'firefox'
184+
'gcc-12'
185+
'gcc-13'
186+
'gcc-14'
187+
'gcc'
188+
'g++-14'
189+
'gfortran-14'
190+
'groff-base'
191+
'kubectl'
192+
'libgl1-mesa-dri'
193+
'microsoft-edge-stable'
194+
'php.*'
195+
'podman'
196+
'powershell'
197+
'skopeo'
198+
'snapd'
199+
'tmux'
200+
)
201+
202+
if isX86; then
203+
packages+=(
204+
'google-chrome-stable'
205+
'google-cloud-cli'
206+
)
207+
fi
208+
209+
sudo apt-get purge -y --autoremove --fix-missing "${packages[@]}"
97210

211+
echo "=> apt-get autoremove"
98212
sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed"
99213
sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed failed"
100214
}
101215

102-
# Remove Docker images
103-
cleanDocker() {
104-
echo "Removing the following docker images:"
105-
sudo docker image ls
106-
echo "Removing docker images..."
107-
sudo docker image prune --all --force || true
108-
}
109-
110-
# Remove Swap storage
111-
cleanSwap() {
112-
sudo swapoff -a || true
113-
sudo rm -rf /mnt/swapfile || true
114-
free -h
115-
}
116-
117-
# Display initial disk space stats
118-
119-
AVAILABLE_INITIAL=$(getAvailableSpace)
216+
removePythonPackages() {
217+
local packages=(
218+
)
120219

121-
printDF "BEFORE CLEAN-UP:"
122-
echo ""
123-
124-
removeDir /usr/local/lib/android
125-
removeDir /usr/share/dotnet
220+
if isX86; then
221+
packages+=(
222+
'ansible-core'
223+
)
224+
fi
126225

127-
# Haskell runtime
128-
removeDir /opt/ghc
129-
removeDir /usr/local/.ghcup
226+
for p in "${packages[@]}"; do
227+
sudo pipx uninstall "$p"
228+
done
229+
}
130230

131-
execAndMeasureSpaceChange cleanPackages "Large misc. packages"
132-
execAndMeasureSpaceChange cleanDocker "Docker images"
133-
execAndMeasureSpaceChange cleanSwap "Swap storage"
231+
main() {
232+
printDF "BEFORE CLEAN-UP:"
134233

135-
# Output saved space statistic
136-
echo ""
137-
printDF "AFTER CLEAN-UP:"
234+
execAndMeasure "Unused packages" cleanPackages
235+
execAndMeasure "Node modules" removeNodeModules
236+
execAndMeasure "Python Packages" removePythonPackages
237+
removeUnusedDirsAndFiles
138238

139-
echo ""
140-
echo ""
239+
printDF "AFTER CLEAN-UP:"
240+
}
141241

142-
printSavedSpace "$AVAILABLE_INITIAL" "Total saved"
242+
execAndMeasure "Total" main

0 commit comments

Comments
 (0)