Skip to content

Commit da7f3b8

Browse files
wudidapaopaowudidapaopao
authored
Verify SQLite dependencies (#392)
* test(ci): remove sqlite deps * test: update macos arm runner * test: update macos arm runner * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update workflow * test: update build.sh * test: update build.sh * test: update build.sh
1 parent 5a1cd0a commit da7f3b8

File tree

6 files changed

+276
-28
lines changed

6 files changed

+276
-28
lines changed

‎.github/workflows/build_linux_arm64_wheels-gh.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,37 @@ jobs:
3232
libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
3333
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
3434
libffi-dev liblzma-dev
35+
- name: Scan SQLite vulnerabilities with grype
36+
run: |
37+
# Install grype and required tools
38+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
39+
sudo apt-get update && sudo apt-get install -y jq lsb-release
40+
41+
# Detect OS distribution info
42+
DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
43+
DISTRO_VERSION=$(lsb_release -sr)
44+
echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
45+
46+
# Update grype vulnerability database
47+
grype db update
48+
49+
# Check SQLite vulnerabilities in installed packages
50+
echo "Scanning SQLite packages for vulnerabilities..."
51+
GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
52+
echo "Raw grype output:"
53+
echo "$GRYPE_RAW_OUTPUT"
54+
55+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
56+
57+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
58+
echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
59+
echo "SQLite vulnerability details:"
60+
echo "$SQLITE_SCAN_OUTPUT"
61+
exit 1
62+
else
63+
echo "✅ No SQLite vulnerabilities found"
64+
fi
65+
continue-on-error: false
3566
- name: Setup pyenv
3667
run: |
3768
curl https://pyenv.run | bash
@@ -135,6 +166,38 @@ jobs:
135166
bash ./chdb/build/build_static_lib.sh
136167
pyenv shell --unset
137168
continue-on-error: false
169+
- name: Scan chdb libraries with grype
170+
run: |
171+
echo "Scanning chdb libraries for vulnerabilities..."
172+
173+
# Files to scan
174+
FILES_TO_SCAN=""
175+
[ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
176+
[ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
177+
FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
178+
179+
SQLITE_VULNERABILITIES_FOUND=false
180+
181+
for file in $FILES_TO_SCAN; do
182+
if [ -f "$file" ]; then
183+
echo "=== Scanning $file ==="
184+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
185+
echo "$SCAN_OUTPUT"
186+
187+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
188+
echo "❌ SQLite vulnerability found in $file"
189+
SQLITE_VULNERABILITIES_FOUND=true
190+
fi
191+
fi
192+
done
193+
194+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
195+
echo "❌ SQLite vulnerabilities detected in chdb libraries!"
196+
exit 1
197+
else
198+
echo "✅ No SQLite vulnerabilities found in chdb libraries"
199+
fi
200+
continue-on-error: false
138201
- name: Run libchdb stub in examples dir
139202
run: |
140203
bash -x ./examples/runStub.sh

‎.github/workflows/build_linux_x86_wheels.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,37 @@ jobs:
3232
libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
3333
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
3434
libffi-dev liblzma-dev
35+
- name: Scan SQLite vulnerabilities with grype
36+
run: |
37+
# Install grype and required tools
38+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
39+
sudo apt-get update && sudo apt-get install -y jq lsb-release
40+
41+
# Detect OS distribution info
42+
DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
43+
DISTRO_VERSION=$(lsb_release -sr)
44+
echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
45+
46+
# Update grype vulnerability database
47+
grype db update
48+
49+
# Check SQLite vulnerabilities in installed packages
50+
echo "Scanning SQLite packages for vulnerabilities..."
51+
GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
52+
echo "Raw grype output:"
53+
echo "$GRYPE_RAW_OUTPUT"
54+
55+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
56+
57+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
58+
echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
59+
echo "SQLite vulnerability details:"
60+
echo "$SQLITE_SCAN_OUTPUT"
61+
exit 1
62+
else
63+
echo "✅ No SQLite vulnerabilities found"
64+
fi
65+
continue-on-error: false
3566
- name: Setup pyenv
3667
run: |
3768
curl https://pyenv.run | bash
@@ -135,6 +166,38 @@ jobs:
135166
bash ./chdb/build/build_static_lib.sh
136167
pyenv shell --unset
137168
continue-on-error: false
169+
- name: Scan chdb libraries with grype
170+
run: |
171+
echo "Scanning chdb libraries for vulnerabilities..."
172+
173+
# Files to scan
174+
FILES_TO_SCAN=""
175+
[ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
176+
[ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
177+
FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
178+
179+
SQLITE_VULNERABILITIES_FOUND=false
180+
181+
for file in $FILES_TO_SCAN; do
182+
if [ -f "$file" ]; then
183+
echo "=== Scanning $file ==="
184+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
185+
echo "$SCAN_OUTPUT"
186+
187+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
188+
echo "❌ SQLite vulnerability found in $file"
189+
SQLITE_VULNERABILITIES_FOUND=true
190+
fi
191+
fi
192+
done
193+
194+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
195+
echo "❌ SQLite vulnerabilities detected in chdb libraries!"
196+
exit 1
197+
else
198+
echo "✅ No SQLite vulnerabilities found in chdb libraries"
199+
fi
200+
continue-on-error: false
138201
- name: Run libchdb stub in examples dir
139202
run: |
140203
bash -x ./examples/runStub.sh

‎.github/workflows/build_macos_arm64_wheels.yml‎

Lines changed: 84 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -76,30 +76,57 @@ jobs:
7676
uname -a
7777
export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
7878
brew update
79-
brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
79+
brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd
80+
brew install openssl@3 || echo "OpenSSL install failed, continuing..."
8081
brew install --ignore-dependencies llvm@19
81-
brew install git ninja libtool gettext gcc binutils grep findutils nasm
82-
brew install --build-from-source ccache
82+
brew install git ninja libtool gettext binutils grep findutils nasm
83+
# brew install gcc || echo "GCC install failed, continuing..."
84+
# brew install ccache || echo "ccache installation failed, continuing without it"
8385
brew install go
8486
cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
8587
export PATH=$(brew --prefix llvm@19)/bin:$PATH
8688
which clang++
8789
clang++ --version
8890
which go
8991
go version
90-
ccache -s
92+
ccache -s || echo "ccache not available yet"
93+
- name: Scan SQLite vulnerabilities with grype
94+
run: |
95+
# Install grype
96+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
97+
98+
# Update grype vulnerability database
99+
grype db update
100+
101+
# Check SQLite vulnerabilities in Homebrew packages
102+
echo "Scanning SQLite packages for vulnerabilities..."
103+
GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true)
104+
echo "Raw grype output:"
105+
echo "$GRYPE_RAW_OUTPUT"
106+
107+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
108+
109+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
110+
echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
111+
echo "SQLite vulnerability details:"
112+
echo "$SQLITE_SCAN_OUTPUT"
113+
exit 1
114+
else
115+
echo "✅ No SQLite vulnerabilities found"
116+
fi
117+
continue-on-error: false
91118
- uses: actions/checkout@v3
92119
with:
93120
fetch-depth: 0
94121
- name: Update submodules
95122
run: |
96123
git submodule update --init --recursive --jobs 4
97-
- name: ccache
98-
uses: hendrikmuhs/[email protected]
99-
with:
100-
key: macos-13-xlarge
101-
max-size: 5G
102-
append-timestamp: true
124+
# - name: ccache
125+
# uses: hendrikmuhs/[email protected]
126+
# with:
127+
# key: macos-13-xlarge
128+
# max-size: 5G
129+
# append-timestamp: true
103130
- name: Run chdb/build.sh
104131
timeout-minutes: 600
105132
run: |
@@ -138,24 +165,56 @@ jobs:
138165
cat buildlib/pychdb_cmd.sh
139166
echo "libchdb_cmd.sh:"
140167
cat buildlib/libchdb_cmd.sh
141-
- name: Run libchdb stub in examples dir
142-
run: |
143-
bash -x ./examples/runStub.sh
144-
- name: Keep killall ccache and wait for ccache to finish
145-
if: always()
168+
- name: Scan chdb libraries with grype
146169
run: |
147-
sleep 60
148-
while ps -ef | grep ccache | grep -v grep; do \
149-
killall ccache; \
150-
sleep 10; \
170+
echo "Scanning chdb libraries for vulnerabilities..."
171+
172+
# Files to scan
173+
FILES_TO_SCAN=""
174+
[ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
175+
[ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
176+
FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
177+
178+
SQLITE_VULNERABILITIES_FOUND=false
179+
180+
for file in $FILES_TO_SCAN; do
181+
if [ -f "$file" ]; then
182+
echo "=== Scanning $file ==="
183+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
184+
echo "$SCAN_OUTPUT"
185+
186+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
187+
echo "❌ SQLite vulnerability found in $file"
188+
SQLITE_VULNERABILITIES_FOUND=true
189+
fi
190+
fi
151191
done
152-
- name: Check ccache statistics
192+
193+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
194+
echo "❌ SQLite vulnerabilities detected in chdb libraries!"
195+
exit 1
196+
else
197+
echo "✅ No SQLite vulnerabilities found in chdb libraries"
198+
fi
199+
continue-on-error: false
200+
- name: Run libchdb stub in examples dir
153201
run: |
154-
ccache -s
155-
ls -lh chdb
156-
df -h
157-
env:
158-
CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
202+
bash -x ./examples/runStub.sh
203+
# - name: Keep killall ccache and wait for ccache to finish
204+
# if: always()
205+
# run: |
206+
# sleep 60
207+
# while ps -ef | grep ccache | grep -v grep; do \
208+
# killall ccache || true; \
209+
# sleep 10; \
210+
# done
211+
# - name: Check ccache statistics
212+
# run: |
213+
# ccache -s || echo "ccache not available"
214+
# ls -lh chdb
215+
# df -h
216+
# env:
217+
# CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
159218
- name: Build wheels
160219
run: |
161220
export PATH="$HOME/.pyenv/bin:$PATH"

‎.github/workflows/build_macos_x86_wheels.yml‎

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ on:
2222
jobs:
2323
build_universal_wheel:
2424
name: Build Universal Wheel (macOS x86_64)
25-
runs-on: macos-13
25+
runs-on: macos-14-large
2626
steps:
2727
- name: Setup pyenv
2828
run: |
@@ -79,7 +79,7 @@ jobs:
7979
brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
8080
brew install --ignore-dependencies llvm@19
8181
brew install git ninja libtool gettext gcc binutils grep findutils nasm
82-
brew install --build-from-source ccache
82+
brew install ccache || echo "ccache installation failed, continuing without it"
8383
brew install go
8484
cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
8585
export PATH=$(brew --prefix llvm@19)/bin:$PATH
@@ -88,6 +88,31 @@ jobs:
8888
which go
8989
go version
9090
ccache -s
91+
- name: Scan SQLite vulnerabilities with grype
92+
run: |
93+
# Install grype
94+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
95+
96+
# Update grype vulnerability database
97+
grype db update
98+
99+
# Check SQLite vulnerabilities in Homebrew packages
100+
echo "Scanning SQLite packages for vulnerabilities..."
101+
GRYPE_RAW_OUTPUT=$(grype dir:/usr/local --scope all-layers 2>/dev/null || true)
102+
echo "Raw grype output:"
103+
echo "$GRYPE_RAW_OUTPUT"
104+
105+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
106+
107+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
108+
echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
109+
echo "SQLite vulnerability details:"
110+
echo "$SQLITE_SCAN_OUTPUT"
111+
exit 1
112+
else
113+
echo "✅ No SQLite vulnerabilities found"
114+
fi
115+
continue-on-error: false
91116
- uses: actions/checkout@v3
92117
with:
93118
fetch-depth: 0
@@ -97,7 +122,7 @@ jobs:
97122
- name: ccache
98123
uses: hendrikmuhs/[email protected]
99124
with:
100-
key: macos-13-x86_64
125+
key: macos-14-x86_64
101126
max-size: 5G
102127
append-timestamp: true
103128
- name: Run chdb/build.sh
@@ -139,6 +164,38 @@ jobs:
139164
cat buildlib/pychdb_cmd.sh
140165
echo "libchdb_cmd.sh:"
141166
cat buildlib/libchdb_cmd.sh
167+
- name: Scan chdb libraries with grype
168+
run: |
169+
echo "Scanning chdb libraries for vulnerabilities..."
170+
171+
# Files to scan
172+
FILES_TO_SCAN=""
173+
[ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
174+
[ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
175+
FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
176+
177+
SQLITE_VULNERABILITIES_FOUND=false
178+
179+
for file in $FILES_TO_SCAN; do
180+
if [ -f "$file" ]; then
181+
echo "=== Scanning $file ==="
182+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
183+
echo "$SCAN_OUTPUT"
184+
185+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
186+
echo "❌ SQLite vulnerability found in $file"
187+
SQLITE_VULNERABILITIES_FOUND=true
188+
fi
189+
fi
190+
done
191+
192+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
193+
echo "❌ SQLite vulnerabilities detected in chdb libraries!"
194+
exit 1
195+
else
196+
echo "✅ No SQLite vulnerabilities found in chdb libraries"
197+
fi
198+
continue-on-error: false
142199
- name: Run libchdb stub in examples dir
143200
run: |
144201
bash -x ./examples/runStub.sh

0 commit comments

Comments
 (0)