ci: setup integration testing

avik-pal · avik-pal · commit 63f85a6155e3 · 2025-09-21T13:30:31.000-04:00
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -62,19 +62,13 @@ jobs:
       os: ${{ matrix.os }}
       runtime: ${{ matrix.runtime }}
 
-  # This has been broken for a while, originating from Enzyme.jl
-  # test-assertions:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       version:
-  #         - "1.10"
-  #   uses: ./.github/workflows/CommonCI.yml
-  #   with:
-  #     julia_version: ${{ matrix.version }}
-  #     os: "ubuntu-24.04"
-  #     runtime: "both"
-  #     assertions: true
+  test-assertions:
+    uses: ./.github/workflows/CommonCI.yml
+    with:
+      julia_version: "1.10"
+      os: "ubuntu-24.04"
+      runtime: "both"
+      assertions: true
 
   downgrade:
     strategy:
diff --git a/.github/workflows/CommonCI.yml b/.github/workflows/CommonCI.yml
@@ -35,6 +35,11 @@ on:
         required: false
         default: false
         type: boolean
+      project:
+        description: "Project"
+        required: false
+        default: "."
+        type: string
 
 jobs:
   test:
@@ -130,11 +135,24 @@ jobs:
           end
           Pkg.develop(dev_pks)
         shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
-        # Only in Julia v1.10 we need to install `ReactantCore` manually.
         if: ${{ inputs.julia_version == '1.10' || inputs.julia_version == 'lts' }}
         env:
           JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
 
+      - name: Install Project
+        if: ${{ inputs.project != '.' }}
+        run: |
+          import Pkg
+          PKg.activate(get(ENV, "JULIA_PROJECT", nothing))
+          if VERSION < v"1.11-"
+              Pkg.develop([PackageSpec(; path="."), PackageSpec(; path="lib/ReactantCore")])
+          end
+          Pkg.instantiate()
+        shell: julia --color=yes --code-coverage=user --depwarn=yes {0}
+        env:
+          JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+          JULIA_PROJECT: ${{ inputs.project }}
+
       # Run the tests (pjrt or ifrt or both)
       - name: "Setup Runtime Preferences (PJRT)"
         if: ${{ inputs.runtime == 'pjrt' || inputs.runtime == 'both' }}
@@ -145,8 +163,9 @@ jobs:
           contents: |
             [Reactant]
             xla_runtime = "PJRT"
+
       - name: "Run Tests (PJRT)"
-        if: ${{ inputs.runtime == 'pjrt' || inputs.runtime == 'both' }}
+        if: ${{ (inputs.runtime == 'pjrt' || inputs.runtime == 'both') && inputs.project == '.' }}
         timeout-minutes: 60
         run: |
           import Pkg
@@ -162,6 +181,14 @@ jobs:
           REACTANT_TEST_GROUP: ${{ inputs.test_group }}
           XLA_FLAGS: "--xla_force_host_platform_device_count=12"
           JULIA_DEBUG: "Reactant,Reactant_jll"
+      - name: "Run Integration Tests (PJRT)"
+        if: ${{ (inputs.runtime == 'pjrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        timeout-minutes: 30
+        run: julia --color=yes --project=test/integration/${{ inputs.project }} --threads=auto --check-bounds=yes -O1 test/integration/${{ inputs.project }}/runtests.jl
+        shell: bash
+        env:
+          JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+          JULIA_DEBUG: "Reactant,Reactant_jll"
 
       - name: "Setup Runtime Preferences (IFRT)"
         if: ${{ inputs.runtime == 'ifrt' || inputs.runtime == 'both' }}
@@ -172,8 +199,9 @@ jobs:
           contents: |
             [Reactant]
             xla_runtime = "IFRT"
+
       - name: "Run Tests (IFRT)"
-        if: ${{ inputs.runtime == 'ifrt' || inputs.runtime == 'both' }}
+        if: ${{ (inputs.runtime == 'ifrt' || inputs.runtime == 'both') && inputs.projec == '.' }}
         timeout-minutes: 60
         run: |
           import Pkg
@@ -189,6 +217,14 @@ jobs:
           REACTANT_TEST_GROUP: ${{ inputs.test_group }}
           XLA_FLAGS: "--xla_force_host_platform_device_count=12"
           JULIA_DEBUG: "Reactant,Reactant_jll"
+      - name: "Run Integration Tests (IFRT)"
+        if: ${{ (inputs.runtime == 'ifrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        timeout-minutes: 30
+        run: julia --color=yes --project=test/integration/${{ inputs.project }} --threads=auto --check-bounds=yes -O1 test/integration/${{ inputs.project }}/runtests.jl
+        shell: bash
+        env:
+          JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+          JULIA_DEBUG: "Reactant,Reactant_jll"
 
       - name: "Upload MLIR modules"
         uses: actions/upload-artifact@v4
diff --git a/.github/workflows/Integration.yml b/.github/workflows/Integration.yml
@@ -0,0 +1,55 @@
+name: Integration
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/Integration.yml"
+      - "ext/**"
+      - "lib/**"
+      - "src/**"
+      - "test/**"
+      - "Project.toml"
+  push:
+    branches:
+      - main
+      - release-*
+    tags: ["*"]
+    paths:
+      - ".github/workflows/Integration.yml"
+      - "ext/**"
+      - "lib/**"
+      - "src/**"
+      - "test/**"
+      - "Project.toml"
+
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - CUDA
+          - FancyArrayTypes
+          - FFT
+          - Flux
+          - Lux
+          - NNlib
+          - Python
+        version:
+          - "1.10"
+          - "1.11"
+        os:
+          - ubuntu-latest
+          - linux-x86-ct6e-180-4tpu
+    uses: ./.github/workflows/CommonCI.yml
+    with:
+      julia_version: ${{ matrix.version }}
+      os: ${{ matrix.os }}
+      runtime: "both"
+      project: "test/integration/${{ matrix.project }}"
diff --git a/test/integration/CUDA/Project.toml b/test/integration/CUDA/Project.toml
@@ -6,6 +6,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/FFT/Project.toml b/test/integration/FFT/Project.toml
@@ -5,6 +5,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/FancyArrayTypes/Project.toml b/test/integration/FancyArrayTypes/Project.toml
@@ -8,6 +8,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/Flux/Project.toml b/test/integration/Flux/Project.toml
@@ -6,6 +6,3 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/Lux/Project.toml b/test/integration/Lux/Project.toml
@@ -13,6 +13,3 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/NNlib/Project.toml b/test/integration/NNlib/Project.toml
@@ -6,6 +6,3 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [sources]
 Reactant = {path = "../../.."}
-
-[extras]
-Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/Python/Project.toml b/test/integration/Python/Project.toml
@@ -2,3 +2,6 @@
 PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
