Skip to content

Commit be34f46

Browse files
raptorsunraptorsun
committed
konflux: hermetic build
Signed-off-by: Haoyu Sun <[email protected]>
1 parent fa2121a commit be34f46

File tree

9 files changed

+3343
-52
lines changed

9 files changed

+3343
-52
lines changed

‎.tekton/lightspeed-stack-pull-request.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,17 @@ spec:
2727
value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:on-pr-{{revision}}
2828
- name: image-expires-after
2929
value: 5d
30+
# todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
3031
- name: build-platforms
3132
value:
3233
- linux/x86_64
34+
# todo: change on-push pipeline,too
35+
- name: build-source-image
36+
value: 'true'
37+
- name: prefetch-input
38+
value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true"}]'
39+
- name: hermetic
40+
value: 'true'
3341
- name: dockerfile
3442
value: Containerfile
3543
pipelineSpec:

‎Containerfile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,12 @@ COPY ${LSC_SOURCE_DIR}/src ./src
2525
COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
2626

2727
# Bundle additional dependencies for library mode.
28-
RUN uv sync --locked --no-dev --group llslibdev
28+
# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
29+
RUN if [ -f /cachi2/cachi2.env ]; then \
30+
. /cachi2/cachi2.env && pip install -r requirements.txt; \
31+
else \
32+
uv sync --locked --no-dev --group llslibdev; \
33+
fi
2934

3035
# Explicitly remove some packages to mitigate some CVEs
3136
# - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

‎pyproject.toml‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,23 @@ Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
8686
name = "pytorch-cpu"
8787
url = "https://download.pytorch.org/whl/cpu"
8888
explicit = true
89+
90+
[[tool.uv.index]]
91+
name = "pypi-default"
92+
url = "https://pypi.org/simple"
93+
explicit = true
8994
[tool.uv.sources]
90-
torch = [{ index = "pytorch-cpu" }]
95+
torch = [
96+
{ index = "pytorch-cpu", group = "llslibdev" },
97+
{ index = "pypi-default", group = "llslibdev-hermetic" }
98+
]
99+
[tool.uv]
100+
conflicts = [
101+
[
102+
{ group = "llslibdev" },
103+
{ group = "llslibdev-hermetic" },
104+
],
105+
]
91106

92107
[dependency-groups]
93108
dev = [
@@ -160,6 +175,54 @@ llslibdev = [
160175
"blobfile>=3.0.0",
161176
"psutil>=7.0.0",
162177
]
178+
llslibdev-hermetic = [
179+
# the same as llslibdev, just using default index.
180+
"matplotlib>=3.10.0",
181+
"pillow>=11.1.0",
182+
"pandas>=2.2.3",
183+
"scikit-learn>=1.5.2",
184+
"psycopg2-binary>=2.9.10",
185+
# API eval: inline::meta-reference
186+
"tree_sitter>=0.24.0",
187+
"pythainlp>=3.0.10",
188+
"langdetect>=1.0.9",
189+
"emoji>=2.1.0",
190+
"nltk>=3.8.1",
191+
# API inference: remote::gemini
192+
"litellm>=1.75.5.post1",
193+
# API inference: inline::sentence-transformers
194+
"sentence-transformers>=5.0.0",
195+
# API vector_io: inline::faiss
196+
"faiss-cpu>=1.11.0",
197+
# API scoring: inline::basic
198+
"requests>=2.32.4",
199+
# API datasetio: inline::localfs
200+
"aiosqlite>=0.21.0",
201+
# API datasetio: remote::huggingface
202+
"datasets>=3.6.0",
203+
# API telemetry: inline::meta-reference
204+
"opentelemetry-sdk>=1.34.1",
205+
"opentelemetry-exporter-otlp>=1.34.1",
206+
# API tool_runtime: inline::rag-runtime
207+
"transformers>=4.34.0",
208+
"numpy==2.2.6",
209+
# API tool_runtime: remote::model-context-protocol
210+
"mcp>=1.9.4",
211+
# API post_training: inline::huggingface
212+
"torch==2.7.1",
213+
"trl>=0.18.2",
214+
"peft>=0.15.2",
215+
# Other
216+
"autoevals>=0.0.129",
217+
"fire>=0.7.0",
218+
"opentelemetry-instrumentation>=0.55b0",
219+
"blobfile>=3.0.0",
220+
"psutil>=7.0.0",
221+
]
222+
tool-hermetic = [
223+
"uv==0.8.15",
224+
"pdm>=2.21.0",
225+
]
163226

164227
build = [
165228
"build>=1.2.2.post1",

‎requirements.hermetic.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
uv==0.8.15
2+
pdm>=2.21.0

‎requirements.txt‎

Lines changed: 2555 additions & 0 deletions
Large diffs are not rendered by default.

‎rpms.in.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
packages: [gcc, jq, patch]
2+
contentOrigin:
3+
repofiles: ["./ubi.repo"]
4+
arches: [x86_64, aarch64]

‎rpms.lock.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
---
2+
lockfileVersion: 1
3+
lockfileVendor: redhat
4+
arches:
5+
- arch: aarch64
6+
packages:
7+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.aarch64.rpm
8+
repoid: ubi-9-appstream-rpms
9+
size: 222582
10+
checksum: sha256:bc2305dad655ddb94f966158112efd6cefa6824d5aa2e80f63881f16cee74598
11+
name: oniguruma
12+
evr: 6.9.6-1.el9.5
13+
sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm
14+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/j/jq-1.6-17.el9_6.2.aarch64.rpm
15+
repoid: ubi-9-baseos-rpms
16+
size: 185443
17+
checksum: sha256:46c953403e2da3202cb0509f4895afacbbfa95f840b4e228dc4c72c1523b8aae
18+
name: jq
19+
evr: 1.6-17.el9_6.2
20+
sourcerpm: jq-1.6-17.el9_6.2.src.rpm
21+
source: []
22+
module_metadata: []
23+
- arch: x86_64
24+
packages:
25+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/o/oniguruma-6.9.6-1.el9.5.x86_64.rpm
26+
repoid: ubi-9-appstream-rpms
27+
size: 226331
28+
checksum: sha256:6c884cc2216e5b4699ebd8cde27b39e99532520b367f645ed6cc660d081916dc
29+
name: oniguruma
30+
evr: 6.9.6-1.el9.5
31+
sourcerpm: oniguruma-6.9.6-1.el9.5.src.rpm
32+
- url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/j/jq-1.6-17.el9_6.2.x86_64.rpm
33+
repoid: ubi-9-baseos-rpms
34+
size: 191681
35+
checksum: sha256:d3c6d74db82f6c55533f2d9798d2d4e44988d212880b5b2afd855a43fe2b17d9
36+
name: jq
37+
evr: 1.6-17.el9_6.2
38+
sourcerpm: jq-1.6-17.el9_6.2.src.rpm
39+
source: []
40+
module_metadata: []

‎ubi.repo‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
[ubi-9-baseos-rpms]
2+
name = Red Hat Universal Base Image 9 (RPMs) - BaseOS
3+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/os
4+
enabled = 1
5+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
6+
gpgcheck = 1
7+
8+
[ubi-9-baseos-debug-rpms]
9+
name = Red Hat Universal Base Image 9 (Debug RPMs) - BaseOS
10+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/debug
11+
enabled = 0
12+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
13+
gpgcheck = 1
14+
15+
[ubi-9-baseos-source-rpms]
16+
name = Red Hat Universal Base Image 9 (Source RPMs) - BaseOS
17+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/baseos/source/SRPMS
18+
enabled = 0
19+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
20+
gpgcheck = 1
21+
22+
[ubi-9-appstream-rpms]
23+
name = Red Hat Universal Base Image 9 (RPMs) - AppStream
24+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/os
25+
enabled = 1
26+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
27+
gpgcheck = 1
28+
29+
[ubi-9-appstream-debug-rpms]
30+
name = Red Hat Universal Base Image 9 (Debug RPMs) - AppStream
31+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/debug
32+
enabled = 0
33+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
34+
gpgcheck = 1
35+
36+
[ubi-9-appstream-source-rpms]
37+
name = Red Hat Universal Base Image 9 (Source RPMs) - AppStream
38+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/appstream/source/SRPMS
39+
enabled = 0
40+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
41+
gpgcheck = 1
42+
43+
[ubi-9-codeready-builder-rpms]
44+
name = Red Hat Universal Base Image 9 (RPMs) - CodeReady Builder
45+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/os
46+
enabled = 1
47+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
48+
gpgcheck = 1
49+
50+
[ubi-9-codeready-builder-debug-rpms]
51+
name = Red Hat Universal Base Image 9 (Debug RPMs) - CodeReady Builder
52+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/debug
53+
enabled = 0
54+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
55+
gpgcheck = 1
56+
57+
[ubi-9-codeready-builder-source-rpms]
58+
name = Red Hat Universal Base Image 9 (Source RPMs) - CodeReady Builder
59+
baseurl = https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/$basearch/codeready-builder/source/SRPMS
60+
enabled = 0
61+
gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
62+
gpgcheck = 1

0 commit comments

Comments
 (0)