konflux: hermetic build

Signed-off-by: Haoyu Sun <[email protected]>

Author: raptorsun

.tekton/lightspeed-stack-pull-request.yaml

@@ -27,9 +27,17 @@ spec:
     value: quay.io/redhat-user-workloads/lightspeed-core-tenant/lightspeed-stack:on-pr-{{revision}}
   - name: image-expires-after
     value: 5d
+  # todo: add arm64. refer to https://konflux.pages.redhat.com/docs/users/getting-started/multi-platform-builds.html#arm64-2
   - name: build-platforms
     value:
     - linux/x86_64
+  # todo: change on-push pipeline,too
+  - name: build-source-image
+    value: 'true'
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "."}, {"type": "pip", "path": ".", "allow_binary": "true"}]'
+  - name: hermetic
+    value: 'true'
   - name: dockerfile
     value: Containerfile
   pipelineSpec:

Containerfile

@@ -22,10 +22,15 @@ RUN pip3.12 install "uv==0.8.15"
 # Add explicit files and directories
 # (avoid accidental inclusion of local directories or env files or credentials)
 COPY ${LSC_SOURCE_DIR}/src ./src
-COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ./
+COPY ${LSC_SOURCE_DIR}/pyproject.toml ${LSC_SOURCE_DIR}/LICENSE ${LSC_SOURCE_DIR}/README.md ${LSC_SOURCE_DIR}/uv.lock ${LSC_SOURCE_DIR}/requirements.txt ./
 
 # Bundle additional dependencies for library mode.
-RUN uv sync --locked --no-dev --group llslibdev
+# Source cachi2 environment for hermetic builds if available, otherwise use normal installation
+RUN if [ -f /cachi2/cachi2.env ]; then \
+        . /cachi2/cachi2.env && cat /cachi2/cachi2.env  && uv venv && uv pip sync --group llslibdev-hermetic pyproject.toml; \
+    else \
+        uv sync --locked --no-dev --group llslibdev; \
+    fi
 
 # Explicitly remove some packages to mitigate some CVEs
 # - GHSA-wj6h-64fc-37mp: python-ecdsa package won't fix it upstream.

pyproject.toml

@@ -86,8 +86,23 @@ Issues = "https://github.com/lightspeed-core/lightspeed-stack/issues"
 name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
+
+[[tool.uv.index]]
+name = "pypi-default"
+url = "https://pypi.org/simple"
+explicit = true
 [tool.uv.sources]
-torch = [{ index = "pytorch-cpu" }]
+torch = [
+    { index = "pytorch-cpu", group = "llslibdev" },
+    { index = "pypi-default", group = "llslibdev-hermetic" }
+]
+[tool.uv]
+conflicts = [
+    [
+      { group = "llslibdev" },
+      { group = "llslibdev-hermetic" },
+    ],
+]
 
 [dependency-groups]
 dev = [
@@ -160,6 +175,54 @@ llslibdev = [
     "blobfile>=3.0.0",
     "psutil>=7.0.0",
 ]
+llslibdev-hermetic = [
+    # the same as llslibdev, just using default index.
+    "matplotlib>=3.10.0",
+    "pillow>=11.1.0",
+    "pandas>=2.2.3",
+    "scikit-learn>=1.5.2",
+    "psycopg2-binary>=2.9.10",
+    # API eval: inline::meta-reference
+    "tree_sitter>=0.24.0",
+    "pythainlp>=3.0.10",
+    "langdetect>=1.0.9",
+    "emoji>=2.1.0",
+    "nltk>=3.8.1",
+    # API inference: remote::gemini
+    "litellm>=1.75.5.post1",
+    # API inference: inline::sentence-transformers
+    "sentence-transformers>=5.0.0",
+    # API vector_io: inline::faiss
+    "faiss-cpu>=1.11.0",
+    # API scoring: inline::basic
+    "requests>=2.32.4",
+    # API datasetio: inline::localfs
+    "aiosqlite>=0.21.0",
+    # API datasetio: remote::huggingface
+    "datasets>=3.6.0",
+    # API telemetry: inline::meta-reference
+    "opentelemetry-sdk>=1.34.1",
+    "opentelemetry-exporter-otlp>=1.34.1",
+    # API tool_runtime: inline::rag-runtime
+    "transformers>=4.34.0",
+    "numpy==2.2.6",
+    # API tool_runtime: remote::model-context-protocol
+    "mcp>=1.9.4",
+    # API post_training: inline::huggingface
+    "torch==2.7.1",
+    "trl>=0.18.2",
+    "peft>=0.15.2",
+    # Other
+    "autoevals>=0.0.129",
+    "fire>=0.7.0",
+    "opentelemetry-instrumentation>=0.55b0",
+    "blobfile>=3.0.0",
+    "psutil>=7.0.0",
+]
+tool-hermetic = [
+    "uv==0.8.15",
+    "pdm>=2.21.0",
+]
 
 build = [
     "build>=1.2.2.post1",

requirements.hermetic.txt

@@ -0,0 +1,2 @@
+uv==0.8.15
+pdm>=2.21.0

requirements.txt

@@ -0,0 +1,4 @@
+packages: [gcc, jq, patch]
+contentOrigin:
+  repofiles: ["./ubi.repo"]
+arches: [x86_64, aarch64]