diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 44f8b50dd7..7ba121da05 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,15 +1,11 @@
 steps:
   - group: ":test_tube: Tests"
     steps:
-      - label: ":julia: :linux: CUDA Julia v{{matrix.version}} -- {{matrix.group}} -- {{matrix.runtime}}"
+      - label: ":julia: :linux: CUDA Julia v{{matrix.version}} -- {{matrix.runtime}}"
         matrix:
           setup:
             version:
               - "1.10"
-            group:
-              - core
-              - neural_networks
-              - integration
             runtime:
               - "PJRT"
               - "IFRT"
@@ -41,7 +37,6 @@ steps:
           queue: "juliagpu"
           cuda: "*"
         env:
-          REACTANT_TEST_GROUP: "{{matrix.group}}"
           JULIA_DEBUG: "Reactant,Reactant_jll"
           CUDA_VISIBLE_DEVICES: 0
           REACTANT_BACKEND_GROUP: "GPU"
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
index b60f801cbc..e43ccaa786 100644
--- a/.github/workflows/CI.yml
+++ b/.github/workflows/CI.yml
@@ -48,10 +48,6 @@ jobs:
           - macOS-latest
           - windows-latest
           - linux-x86-ct6e-180-4tpu
-        test_group:
-          - core
-          - neural_networks
-          - integration
         runtime:
           - "pjrt"
           - "ifrt"
@@ -65,35 +61,19 @@ jobs:
       julia_version: ${{ matrix.version }}
       os: ${{ matrix.os }}
       runtime: ${{ matrix.runtime }}
-      test_group: ${{ matrix.test_group }}
 
-  # This has been broken for a while, originating from CUDA.jl
-  # test-assertions:
-  #   strategy:
-  #     fail-fast: false
-  #     matrix:
-  #       version:
-  #         - "1.10"
-  #       test_group:
-  #         - core
-  #         - neural_networks
-  #         - integration
-  #   uses: ./.github/workflows/CommonCI.yml
-  #   with:
-  #     julia_version: ${{ matrix.version }}
-  #     os: "ubuntu-24.04"
-  #     runtime: "both"
-  #     assertions: true
-  #     test_group: ${{ matrix.test_group }}
+  test-assertions:
+    uses: ./.github/workflows/CommonCI.yml
+    with:
+      julia_version: "1.10"
+      os: "ubuntu-24.04"
+      runtime: "both"
+      assertions: true
 
   downgrade:
     strategy:
       fail-fast: false
       matrix:
-        test_group:
-          - core
-          - neural_networks
-          - integration
         runtime:
           - "pjrt"
           - "ifrt"
@@ -102,5 +82,54 @@ jobs:
       julia_version: "1.10"
       os: "ubuntu-24.04"
       runtime: ${{ matrix.runtime }}
-      test_group: ${{ matrix.test_group }}
       downgrade_testing: true
+
+  integration-cpu:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - FancyArrayTypes
+          - FFT
+          - Python
+        version:
+          - "1.10"
+          - "1.11"
+    uses: ./.github/workflows/CommonCI.yml
+    with:
+      julia_version: ${{ matrix.version }}
+      os: "ubuntu-latest"
+      runtime: "both"
+      project: "test/integration/${{ matrix.project }}"
+
+  integration:
+    strategy:
+      fail-fast: false
+      matrix:
+        project:
+          - CUDA
+          - Flux
+          - Lux
+          - NNlib
+        version:
+          - "1.10"
+          - "1.11"
+        os:
+          - ubuntu-latest
+          - linux-x86-ct6e-180-4tpu
+        exclude:
+          - os: linux-x86-ct6e-180-4tpu
+            version: "1.10"
+        include:
+          - project: CUDA
+            os: linux-x86-a2-48-a100-4gpu
+            version: "1.10"
+          - project: CUDA
+            os: linux-x86-a2-48-a100-4gpu
+            version: "1.11"
+    uses: ./.github/workflows/CommonCI.yml
+    with:
+      julia_version: ${{ matrix.version }}
+      os: ${{ matrix.os }}
+      runtime: "both"
+      project: "test/integration/${{ matrix.project }}"
diff --git a/.github/workflows/CommonCI.yml b/.github/workflows/CommonCI.yml
index 5bb654bf5d..57f6c1ab6d 100644
--- a/.github/workflows/CommonCI.yml
+++ b/.github/workflows/CommonCI.yml
@@ -35,6 +35,11 @@ on:
         required: false
         default: false
         type: boolean
+      project:
+        description: "Project"
+        required: false
+        default: "."
+        type: string
 
 jobs:
   test:
@@ -70,7 +75,7 @@ jobs:
       - uses: julia-actions/cache@v2
         id: julia-cache
         with:
-          cache-name: julia-cache;workflow=${{ inputs.julia_version }}-${{ inputs.os }}-${{ inputs.runtime }}-assertions=${{ inputs.assertions }}-${{ github.event_name }}-${{ inputs.test_group }}-${{ inputs.downgrade_testing }}-${{ inputs.localjll }}
+          cache-name: julia-cache;workflow=${{ inputs.julia_version }}-${{ inputs.os }}-${{ inputs.runtime }}-assertions=${{ inputs.assertions }}-${{ github.event_name }}-${{ inputs.test_group }}-${{ inputs.downgrade_testing }}-${{ inputs.localjll }}-${{ inputs.project }}
 
       - uses: julia-actions/julia-downgrade-compat@v2
         if: ${{ inputs.downgrade_testing }}
@@ -132,9 +137,25 @@ jobs:
           end
           Pkg.develop(dev_pks)
         shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0}
-        # Only in Julia v1.10 we need to install `ReactantCore` manually.
         if: ${{ inputs.julia_version == '1.10' || inputs.julia_version == 'lts' }}
 
+      - name: Install Project
+        if: ${{ inputs.project != '.' }}
+        run: |
+          import Pkg
+          Pkg.activate(get(ENV, "JULIA_PROJECT", nothing))
+          if VERSION < v"1.11-"
+              Pkg.develop([
+                Pkg.PackageSpec(; path="."),
+                Pkg.PackageSpec(; path="lib/ReactantCore")
+              ])
+          end
+          Pkg.instantiate()
+        shell: julia --color=yes --code-coverage=user --depwarn=yes {0}
+        env:
+          JULIA_PKG_SERVER_REGISTRY_PREFERENCE: eager
+          JULIA_PROJECT: ${{ inputs.project }}
+
       # Run the tests (pjrt or ifrt or both)
       - name: "Setup Runtime Preferences (PJRT)"
         if: ${{ inputs.runtime == 'pjrt' || inputs.runtime == 'both' }}
@@ -145,8 +166,18 @@ jobs:
           contents: |
             [Reactant]
             xla_runtime = "PJRT"
+      - name: "Setup Runtime Preferences for ${{ inputs.project }} (PJRT)"
+        if: ${{ (inputs.runtime == 'pjrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        uses: "DamianReeves/write-file-action@master"
+        with:
+          path: ${{ inputs.project }}/LocalPreferences.toml
+          write-mode: "overwrite"
+          contents: |
+            [Reactant]
+            xla_runtime = "PJRT"
+
       - name: "Run Tests (PJRT)"
-        if: ${{ inputs.runtime == 'pjrt' || inputs.runtime == 'both' }}
+        if: ${{ (inputs.runtime == 'pjrt' || inputs.runtime == 'both') && inputs.project == '.' }}
         timeout-minutes: 120
         run: |
           import Pkg
@@ -159,6 +190,11 @@ jobs:
         env:
           ALLOW_RERESOLVE: ${{ !inputs.downgrade_testing }}
           REACTANT_TEST_GROUP: ${{ inputs.test_group }}
+      - name: "Run ${{ inputs.project }} Integration Tests (PJRT)"
+        if: ${{ (inputs.runtime == 'pjrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        timeout-minutes: 30
+        run: julia --color=yes --project=${{ inputs.project }} --threads=auto --check-bounds=yes -O1 ${{ inputs.project }}/runtests.jl
+        shell: bash
 
       - name: "Setup Runtime Preferences (IFRT)"
         if: ${{ inputs.runtime == 'ifrt' || inputs.runtime == 'both' }}
@@ -169,8 +205,18 @@ jobs:
           contents: |
             [Reactant]
             xla_runtime = "IFRT"
+      - name: "Setup Runtime Preferences for ${{ inputs.project }} (IFRT)"
+        if: ${{ (inputs.runtime == 'ifrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        uses: "DamianReeves/write-file-action@master"
+        with:
+          path: ${{ inputs.project }}/LocalPreferences.toml
+          write-mode: "overwrite"
+          contents: |
+            [Reactant]
+            xla_runtime = "IFRT"
+
       - name: "Run Tests (IFRT)"
-        if: ${{ inputs.runtime == 'ifrt' || inputs.runtime == 'both' }}
+        if: ${{ (inputs.runtime == 'ifrt' || inputs.runtime == 'both') && inputs.project == '.' }}
         timeout-minutes: 120
         run: |
           import Pkg
@@ -183,6 +229,11 @@ jobs:
         env:
           ALLOW_RERESOLVE: ${{ !inputs.downgrade_testing }}
           REACTANT_TEST_GROUP: ${{ inputs.test_group }}
+      - name: "Run ${{ inputs.project }} Integration Tests (IFRT)"
+        if: ${{ (inputs.runtime == 'ifrt' || inputs.runtime == 'both') && inputs.project != '.' }}
+        timeout-minutes: 30
+        run: julia --color=yes --project=${{ inputs.project }} --threads=auto --check-bounds=yes -O1 ${{ inputs.project }}/runtests.jl
+        shell: bash
 
       - name: "Upload MLIR modules"
         uses: actions/upload-artifact@v4
diff --git a/ext/ReactantFillArraysExt.jl b/ext/ReactantFillArraysExt.jl
index f0c048581e..8ffc46bc77 100644
--- a/ext/ReactantFillArraysExt.jl
+++ b/ext/ReactantFillArraysExt.jl
@@ -1,6 +1,6 @@
 module ReactantFillArraysExt
 
-using Reactant: Reactant, TracedUtils, TracedRNumber, Ops, Sharding, unwrapped_eltype
+using Reactant: Reactant, TracedRNumber, Sharding, unwrapped_eltype
 using ReactantCore: ReactantCore
 using FillArrays: FillArrays, AbstractFill, Fill, Ones, Zeros, OneElement
 using GPUArraysCore: @allowscalar
diff --git a/test/Project.toml b/test/Project.toml
index 05b1e8906a..8bd2903feb 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -2,29 +2,17 @@
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 DLFP8Types = "f4c16678-4a16-415b-82ef-ed337c5d6c7c"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
 ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
-FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
-FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
 Float8s = "81dfefd7-55b0-40c6-a251-db853704e186"
-Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
 HypothesisTests = "09f84164-cd44-5f33-b23f-e6b0d136a0d5"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
-LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11"
 MethodAnalysis = "85b6ec6f-f7df-4429-9514-a64bcd9ee824"
-NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
-OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"
-Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
 Preferences = "21216c6a-2e73-6563-6e65-726566657250"
-PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Random123 = "74087812-796a-5b5d-8853-05524746bad3"
 SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
@@ -33,32 +21,19 @@ StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
 
 [compat]
 Adapt = "4.1"
 ArrayInterface = "7.17.1"
-CUDA = "5.5"
 DLFP8Types = "0.1"
 Distributions = "0.25"
 Enzyme = "0.13.28"
-FFTW = "1.8"
-FillArrays = "1"
 Float8s = "0.1"
-Flux = "0.15, 0.16"
 Functors = "0.5"
 HypothesisTests = "0.11"
 InteractiveUtils = "1.10"
-KernelAbstractions = "0.9.30"
 LinearAlgebra = "1.10"
-Lux = "1.21"
-LuxLib = "1.11"
-NNlib = "0.9.26"
-OffsetArrays = "1"
-OneHotArrays = "0.2.6"
-Optimisers = "0.4"
 Preferences = "1.4"
-PythonCall = "0.9"
 Random = "1.10"
 Random123 = "1.7"
 SafeTestsets = "0.1"
@@ -67,7 +42,6 @@ StableRNGs = "1"
 Statistics = "1.10"
 StatsBase = "0.34"
 Test = "1.10"
-Zygote = "0.7"
 
 [extras]
 Reactant_jll = "0192cb87-2b54-54ad-80e0-3be72ad8a3c0"
diff --git a/test/integration/special_functions.jl b/test/common_integration/special_functions.jl
similarity index 100%
rename from test/integration/special_functions.jl
rename to test/common_integration/special_functions.jl
diff --git a/test/autodiff.jl b/test/core/autodiff.jl
similarity index 100%
rename from test/autodiff.jl
rename to test/core/autodiff.jl
diff --git a/test/basic.jl b/test/core/basic.jl
similarity index 100%
rename from test/basic.jl
rename to test/core/basic.jl
diff --git a/test/batching.jl b/test/core/batching.jl
similarity index 100%
rename from test/batching.jl
rename to test/core/batching.jl
diff --git a/test/bcast.jl b/test/core/bcast.jl
similarity index 96%
rename from test/bcast.jl
rename to test/core/bcast.jl
index 85f6f4249d..0b163fe526 100644
--- a/test/bcast.jl
+++ b/test/core/bcast.jl
@@ -1,8 +1,5 @@
-using Test
-using Reactant
-using Enzyme, NNlib
-using Reactant.MLIR
-using Statistics
+using Test, Reactant, Enzyme, Statistics
+using Reactant: MLIR
 
 @noinline function no(@nospecialize(x))
     x = @ccall $(Base.@cfunction(identity, Any, (Any,)))(x::Any)::Any
diff --git a/test/buffer_donation.jl b/test/core/buffer_donation.jl
similarity index 100%
rename from test/buffer_donation.jl
rename to test/core/buffer_donation.jl
diff --git a/test/closure.jl b/test/core/closure.jl
similarity index 100%
rename from test/closure.jl
rename to test/core/closure.jl
diff --git a/test/compile.jl b/test/core/compile.jl
similarity index 100%
rename from test/compile.jl
rename to test/core/compile.jl
diff --git a/test/complex.jl b/test/core/complex.jl
similarity index 100%
rename from test/complex.jl
rename to test/core/complex.jl
diff --git a/test/config.jl b/test/core/config.jl
similarity index 100%
rename from test/config.jl
rename to test/core/config.jl
diff --git a/test/constructor.jl b/test/core/constructor.jl
similarity index 100%
rename from test/constructor.jl
rename to test/core/constructor.jl
diff --git a/test/control_flow.jl b/test/core/control_flow.jl
similarity index 100%
rename from test/control_flow.jl
rename to test/core/control_flow.jl
diff --git a/test/custom_number_types.jl b/test/core/custom_number_types.jl
similarity index 100%
rename from test/custom_number_types.jl
rename to test/core/custom_number_types.jl
diff --git a/test/indexing.jl b/test/core/indexing.jl
similarity index 100%
rename from test/indexing.jl
rename to test/core/indexing.jl
diff --git a/test/ir.jl b/test/core/ir.jl
similarity index 100%
rename from test/ir.jl
rename to test/core/ir.jl
diff --git a/test/layout.jl b/test/core/layout.jl
similarity index 100%
rename from test/layout.jl
rename to test/core/layout.jl
diff --git a/test/ops.jl b/test/core/ops.jl
similarity index 100%
rename from test/ops.jl
rename to test/core/ops.jl
diff --git a/test/qa.jl b/test/core/qa.jl
similarity index 100%
rename from test/qa.jl
rename to test/core/qa.jl
diff --git a/test/sorting.jl b/test/core/sorting.jl
similarity index 100%
rename from test/sorting.jl
rename to test/core/sorting.jl
diff --git a/test/struct.jl b/test/core/struct.jl
similarity index 100%
rename from test/struct.jl
rename to test/core/struct.jl
diff --git a/test/tracing.jl b/test/core/tracing.jl
similarity index 100%
rename from test/tracing.jl
rename to test/core/tracing.jl
diff --git a/test/wrapped_arrays.jl b/test/core/wrapped_arrays.jl
similarity index 94%
rename from test/wrapped_arrays.jl
rename to test/core/wrapped_arrays.jl
index a9b303eda5..d7d8dffe66 100644
--- a/test/wrapped_arrays.jl
+++ b/test/core/wrapped_arrays.jl
@@ -1,4 +1,4 @@
-using Reactant, Test, Statistics, NNlib, LinearAlgebra
+using Reactant, Test, Statistics, LinearAlgebra
 
 function view_getindex_1(x)
     x = view(x, 2:3, 1:2, :)
@@ -77,18 +77,6 @@ end
     @test v1 ≈ v2
 end
 
-function btranspose_badjoint(x)
-    x1 = NNlib.batched_transpose(x)
-    x2 = NNlib.batched_adjoint(x)
-    return x1 .+ x2
-end
-
-@testset "batched transpose/adjoint" begin
-    x = rand(4, 2, 3)
-    x_ra = Reactant.to_rarray(x)
-    @test @jit(btranspose_badjoint(x_ra)) ≈ btranspose_badjoint(x)
-end
-
 function bypass_permutedims(x)
     x = PermutedDimsArray(x, (2, 1, 3))  # Don't use permutedims here
     return view(x, 2:3, 1:2, :)
diff --git a/test/cluster_detector.jl b/test/distributed/cluster_detector.jl
similarity index 100%
rename from test/cluster_detector.jl
rename to test/distributed/cluster_detector.jl
diff --git a/test/optimize_comm.jl b/test/distributed/optimize_comm.jl
similarity index 100%
rename from test/optimize_comm.jl
rename to test/distributed/optimize_comm.jl
diff --git a/test/sharding.jl b/test/distributed/sharding.jl
similarity index 100%
rename from test/sharding.jl
rename to test/distributed/sharding.jl
diff --git a/test/integration/CUDA/Project.toml b/test/integration/CUDA/Project.toml
new file mode 100644
index 0000000000..8b1c702ca1
--- /dev/null
+++ b/test/integration/CUDA/Project.toml
@@ -0,0 +1,9 @@
+[deps]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/integration/cuda.jl b/test/integration/CUDA/cuda.jl
similarity index 100%
rename from test/integration/cuda.jl
rename to test/integration/CUDA/cuda.jl
diff --git a/test/integration/kernelabstractions.jl b/test/integration/CUDA/kernelabstractions.jl
similarity index 100%
rename from test/integration/kernelabstractions.jl
rename to test/integration/CUDA/kernelabstractions.jl
diff --git a/test/integration/CUDA/runtests.jl b/test/integration/CUDA/runtests.jl
new file mode 100644
index 0000000000..9c4cbf88a3
--- /dev/null
+++ b/test/integration/CUDA/runtests.jl
@@ -0,0 +1,11 @@
+using Test, SafeTestsets
+
+@testset "CUDA Integration" begin
+    @safetestset "CUDA" begin
+        include("cuda.jl")
+    end
+
+    @safetestset "KernelAbstractions" begin
+        include("kernelabstractions.jl")
+    end
+end
diff --git a/test/integration/FFT/Project.toml b/test/integration/FFT/Project.toml
new file mode 100644
index 0000000000..a93b2fc311
--- /dev/null
+++ b/test/integration/FFT/Project.toml
@@ -0,0 +1,7 @@
+[deps]
+FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/integration/fft.jl b/test/integration/FFT/runtests.jl
similarity index 100%
rename from test/integration/fft.jl
rename to test/integration/FFT/runtests.jl
diff --git a/test/integration/FancyArrayTypes/Project.toml b/test/integration/FancyArrayTypes/Project.toml
new file mode 100644
index 0000000000..dde4fb4800
--- /dev/null
+++ b/test/integration/FancyArrayTypes/Project.toml
@@ -0,0 +1,10 @@
+[deps]
+FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b"
+OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
+OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/integration/fillarrays.jl b/test/integration/FancyArrayTypes/fillarrays.jl
similarity index 100%
rename from test/integration/fillarrays.jl
rename to test/integration/FancyArrayTypes/fillarrays.jl
diff --git a/test/integration/offsetarrays.jl b/test/integration/FancyArrayTypes/offsetarrays.jl
similarity index 88%
rename from test/integration/offsetarrays.jl
rename to test/integration/FancyArrayTypes/offsetarrays.jl
index a8bb4f899e..cf1d47a41f 100644
--- a/test/integration/offsetarrays.jl
+++ b/test/integration/FancyArrayTypes/offsetarrays.jl
@@ -1,10 +1,9 @@
-using Reactant
-using Test
-using OffsetArrays
+using Reactant, Test, OffsetArrays
 
 function scalar_index(x)
     @allowscalar getindex(x, -1, 0)
 end
+
 @testset "OffsetArrays" begin
     A = Float64.(reshape(1:15, 3, 5))
     OA = OffsetArray(A, -1:1, 0:4)
diff --git a/test/integration/onehotarrays.jl b/test/integration/FancyArrayTypes/onehotarrays.jl
similarity index 100%
rename from test/integration/onehotarrays.jl
rename to test/integration/FancyArrayTypes/onehotarrays.jl
diff --git a/test/integration/FancyArrayTypes/runtests.jl b/test/integration/FancyArrayTypes/runtests.jl
new file mode 100644
index 0000000000..d6d92bb013
--- /dev/null
+++ b/test/integration/FancyArrayTypes/runtests.jl
@@ -0,0 +1,15 @@
+using Test
+
+@testset "Fancy Array Types" begin
+    @testset "OffsetArrays" begin
+        include("offsetarrays.jl")
+    end
+
+    @testset "FillArrays" begin
+        include("fillarrays.jl")
+    end
+
+    @testset "OneHotArrays" begin
+        include("onehotarrays.jl")
+    end
+end
diff --git a/test/integration/Flux/Project.toml b/test/integration/Flux/Project.toml
new file mode 100644
index 0000000000..700f2c77d4
--- /dev/null
+++ b/test/integration/Flux/Project.toml
@@ -0,0 +1,9 @@
+[deps]
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/integration/Flux/flux.jl b/test/integration/Flux/flux.jl
new file mode 100644
index 0000000000..b94b5946f2
--- /dev/null
+++ b/test/integration/Flux/flux.jl
@@ -0,0 +1,19 @@
+using Reactant, Flux, Test
+
+@testset "Flux.jl Integration" begin
+    noisy = rand(Float32, 2, 1000)
+    truth = [xor(col[1] > 0.5, col[2] > 0.5) for col in eachcol(noisy)]
+
+    model = Chain(Dense(2 => 3, tanh), BatchNorm(3), Dense(3 => 2), softmax)
+
+    origout = model(noisy)
+
+    cmodel = Reactant.to_rarray(model)
+    cnoisy = Reactant.to_rarray(noisy)
+
+    fn = (a, b) -> a(b)
+    f = @compile fn(cmodel, cnoisy)
+
+    comp = f(cmodel, cnoisy)
+    @test origout ≈ comp atol = 1e-3 rtol = 1e-2
+end
diff --git a/test/integration/Flux/runtests.jl b/test/integration/Flux/runtests.jl
new file mode 100644
index 0000000000..8c05868052
--- /dev/null
+++ b/test/integration/Flux/runtests.jl
@@ -0,0 +1,11 @@
+using Test
+
+@testset "Flux.jl Integration" begin
+    @testset "Zygote Integration" begin
+        include("zygote.jl")
+    end
+
+    @testset "Flux Integration" begin
+        include("flux.jl")
+    end
+end
diff --git a/test/integration/zygote.jl b/test/integration/Flux/zygote.jl
similarity index 100%
rename from test/integration/zygote.jl
rename to test/integration/Flux/zygote.jl
diff --git a/test/integration/Lux/Project.toml b/test/integration/Lux/Project.toml
new file mode 100644
index 0000000000..0f85c564e2
--- /dev/null
+++ b/test/integration/Lux/Project.toml
@@ -0,0 +1,15 @@
+[deps]
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196"
+Lux = "b2108857-7c20-44ae-9111-449ecde12c47"
+LuxLib = "82251201-b29d-42c6-8e01-566dec8acb11"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+OneHotArrays = "0b1bfda6-eb8a-41d2-88d8-f5af5cad476f"
+Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/nn/lux.jl b/test/integration/Lux/lux.jl
similarity index 100%
rename from test/nn/lux.jl
rename to test/integration/Lux/lux.jl
diff --git a/test/nn/luxlib.jl b/test/integration/Lux/luxlib.jl
similarity index 100%
rename from test/nn/luxlib.jl
rename to test/integration/Lux/luxlib.jl
diff --git a/test/integration/optimisers.jl b/test/integration/Lux/optimisers.jl
similarity index 100%
rename from test/integration/optimisers.jl
rename to test/integration/Lux/optimisers.jl
diff --git a/test/integration/Lux/runtests.jl b/test/integration/Lux/runtests.jl
new file mode 100644
index 0000000000..173f2450ab
--- /dev/null
+++ b/test/integration/Lux/runtests.jl
@@ -0,0 +1,15 @@
+using Test
+
+@testset "Lux.jl Integration" begin
+    @testset "LuxLib Primitives" begin
+        include("luxlib.jl")
+    end
+
+    @testset "Lux Integration" begin
+        include("lux.jl")
+    end
+
+    @testset "Optimisers Integration" begin
+        include("optimisers.jl")
+    end
+end
diff --git a/test/integration/NNlib/Project.toml b/test/integration/NNlib/Project.toml
new file mode 100644
index 0000000000..b4b1da67c2
--- /dev/null
+++ b/test/integration/NNlib/Project.toml
@@ -0,0 +1,8 @@
+[deps]
+Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/nn/nnlib.jl b/test/integration/NNlib/runtests.jl
similarity index 98%
rename from test/nn/nnlib.jl
rename to test/integration/NNlib/runtests.jl
index 240c52c155..1436466886 100644
--- a/test/nn/nnlib.jl
+++ b/test/integration/NNlib/runtests.jl
@@ -1,5 +1,4 @@
-using NNlib, Reactant, Enzyme
-using Statistics
+using NNlib, Reactant, Enzyme, Test, Statistics
 
 @testset "Activation Functions" begin
     sumabs2(f, x) = sum(abs2, f.(x))
@@ -793,3 +792,15 @@ end
     hlo = repr(@code_hlo(NNlib.gather(x_ra, idxs_ra)))
     @test !contains(hlo, "i64>")
 end
+
+function btranspose_badjoint(x)
+    x1 = NNlib.batched_transpose(x)
+    x2 = NNlib.batched_adjoint(x)
+    return x1 .+ x2
+end
+
+@testset "batched transpose/adjoint" begin
+    x = rand(4, 2, 3)
+    x_ra = Reactant.to_rarray(x)
+    @test @jit(btranspose_badjoint(x_ra)) ≈ btranspose_badjoint(x)
+end
diff --git a/test/integration/Python/Project.toml b/test/integration/Python/Project.toml
new file mode 100644
index 0000000000..a24e261e4b
--- /dev/null
+++ b/test/integration/Python/Project.toml
@@ -0,0 +1,7 @@
+[deps]
+PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
+Reactant = "3c362404-f566-11ee-1572-e11a4b42c853"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[sources]
+Reactant = {path = "../../.."}
diff --git a/test/integration/python.jl b/test/integration/Python/runtests.jl
similarity index 98%
rename from test/integration/python.jl
rename to test/integration/Python/runtests.jl
index f0f04ae7c8..c51eb6e7b3 100644
--- a/test/integration/python.jl
+++ b/test/integration/Python/runtests.jl
@@ -1,8 +1,6 @@
-using Reactant
+using Reactant, Test
 using Reactant: Ops
 
-using Test
-
 fn(x, y) = sin.(x) .+ cos.(y.x[1:2, :])
 
 # Jax on Github CI dislikes X86 macos
diff --git a/test/nn/flux.jl b/test/nn/flux.jl
deleted file mode 100644
index f4c314bccd..0000000000
--- a/test/nn/flux.jl
+++ /dev/null
@@ -1,25 +0,0 @@
-using Reactant, Flux
-
-@testset "Flux.jl Integration" begin
-    # Generate some data for the XOR problem: vectors of length 2, as columns of a matrix:
-    noisy = rand(Float32, 2, 1000)                                        # 2×1000 Matrix{Float32}
-    truth = [xor(col[1] > 0.5, col[2] > 0.5) for col in eachcol(noisy)]   # 1000-element Vector{Bool}
-
-    # Define our model, a multi-layer perceptron with one hidden layer of size 3:
-    model = Chain(
-        Dense(2 => 3, tanh),   # activation function inside layer
-        BatchNorm(3),
-        Dense(3 => 2),
-        softmax,
-    )
-
-    origout = model(noisy)
-
-    cmodel = Reactant.to_rarray(model)
-    cnoisy = Reactant.to_rarray(noisy)
-
-    f = Reactant.compile((a, b) -> a(b), (cmodel, cnoisy))
-
-    comp = f(cmodel, cnoisy)
-    @test origout ≈ comp atol = 1e-3 rtol = 1e-2
-end
diff --git a/test/runtests.jl b/test/runtests.jl
index 98a02a7de0..fc3a672171 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,66 +1,66 @@
 using Reactant, SafeTestsets, Test
 
-if lowercase(get(ENV, "REACTANT_BACKEND_GROUP", "all")) == "gpu"
+const REACTANT_BACKEND_GROUP = lowercase(get(ENV, "REACTANT_BACKEND_GROUP", "all"))
+
+if REACTANT_BACKEND_GROUP == "gpu"
     Reactant.set_default_backend("gpu")
+elseif REACTANT_BACKEND_GROUP == "tpu"
+    Reactant.set_default_backend("tpu")
+elseif REACTANT_BACKEND_GROUP == "cpu"
+    Reactant.set_default_backend("cpu")
 end
 
-const REACTANT_TEST_GROUP = lowercase(get(ENV, "REACTANT_TEST_GROUP", "all"))
-
 @testset "Reactant.jl Tests" begin
-    if REACTANT_TEST_GROUP == "all" || REACTANT_TEST_GROUP == "core"
-        if Sys.isapple() && haskey(Reactant.XLA.global_backend_state.clients, "metal")
-            @safetestset "Metal Plugin" include("plugins/metal.jl")
-        end
+    @testset "Core" begin
+        @safetestset "Layout" include("core/layout.jl")
+        @safetestset "Tracing" include("core/tracing.jl")
+        @safetestset "Basic" include("core/basic.jl")
+        @safetestset "Constructor" include("core/constructor.jl")
+        @safetestset "Autodiff" include("core/autodiff.jl")
+        @safetestset "Complex" include("core/complex.jl")
+        @safetestset "Broadcast" include("core/bcast.jl")
+        @safetestset "Struct" include("core/struct.jl")
+        @safetestset "Closure" include("core/closure.jl")
+        @safetestset "Compile" include("core/compile.jl")
+        @safetestset "IR" include("core/ir.jl")
+        @safetestset "Buffer Donation" include("core/buffer_donation.jl")
+        @safetestset "Wrapped Arrays" include("core/wrapped_arrays.jl")
+        @safetestset "Control Flow" include("core/control_flow.jl")
+        @safetestset "Sorting" include("core/sorting.jl")
+        @safetestset "Shortcuts to MLIR ops" include("core/ops.jl")
+        @safetestset "Indexing" include("core/indexing.jl")
+        @safetestset "Config" include("core/config.jl")
+        @safetestset "Batching" include("core/batching.jl")
 
-        @safetestset "Layout" include("layout.jl")
-        @safetestset "Tracing" include("tracing.jl")
-        @safetestset "Basic" include("basic.jl")
-        @safetestset "Constructor" include("constructor.jl")
-        @safetestset "Autodiff" include("autodiff.jl")
-        @safetestset "Complex" include("complex.jl")
-        @safetestset "Broadcast" include("bcast.jl")
-        @safetestset "Struct" include("struct.jl")
-        @safetestset "Closure" include("closure.jl")
-        @safetestset "Compile" include("compile.jl")
-        @safetestset "IR" include("ir.jl")
-        @safetestset "Buffer Donation" include("buffer_donation.jl")
-        @safetestset "Wrapped Arrays" include("wrapped_arrays.jl")
-        @safetestset "Control Flow" include("control_flow.jl")
-        @safetestset "Sorting" include("sorting.jl")
-        @safetestset "Shortcuts to MLIR ops" include("ops.jl")
-        @safetestset "Indexing" include("indexing.jl")
         if !Sys.isapple()
-            @safetestset "Custom Number Types" include("custom_number_types.jl")
+            @safetestset "Custom Number Types" include("core/custom_number_types.jl")
         end
-        @safetestset "Sharding" include("sharding.jl")
-        @safetestset "Comm Optimization" include("optimize_comm.jl")
-        @safetestset "Cluster Detection" include("cluster_detector.jl")
-        @safetestset "Config" include("config.jl")
-        @safetestset "Batching" include("batching.jl")
-        @safetestset "QA" include("qa.jl")
     end
 
-    if REACTANT_TEST_GROUP == "all" || REACTANT_TEST_GROUP == "integration"
-        @safetestset "CUDA" include("integration/cuda.jl")
-        @safetestset "KernelAbstractions" include("integration/kernelabstractions.jl")
-        @safetestset "Linear Algebra" include("integration/linear_algebra.jl")
-        @safetestset "OffsetArrays" include("integration/offsetarrays.jl")
-        @safetestset "OneHotArrays" include("integration/onehotarrays.jl")
-        @safetestset "AbstractFFTs" include("integration/fft.jl")
-        @safetestset "SpecialFunctions" include("integration/special_functions.jl")
-        @safetestset "Random" include("integration/random.jl")
-        @safetestset "Python" include("integration/python.jl")
-        @safetestset "Optimisers" include("integration/optimisers.jl")
-        @safetestset "FillArrays" include("integration/fillarrays.jl")
-        @safetestset "Zygote" include("integration/zygote.jl")
+    @testset "Distributed" begin
+        @safetestset "Sharding" include("distributed/sharding.jl")
+        @safetestset "Comm Optimization" include("distributed/optimize_comm.jl")
+        @safetestset "Cluster Detection" include("distributed/cluster_detector.jl")
     end
 
-    if REACTANT_TEST_GROUP == "all" || REACTANT_TEST_GROUP == "neural_networks"
-        @safetestset "NNlib Primitives" include("nn/nnlib.jl")
-        @safetestset "Flux.jl Integration" include("nn/flux.jl")
-        if Sys.islinux()
-            @safetestset "LuxLib Primitives" include("nn/luxlib.jl")
-            @safetestset "Lux Integration" include("nn/lux.jl")
+    @testset "Plugins" begin
+        if Sys.isapple() && haskey(Reactant.XLA.global_backend_state.clients, "metal")
+            @safetestset "Metal Plugin" include("plugins/metal.jl")
         end
     end
+
+    @testset "Standard Libraries" begin
+        @safetestset "Linear Algebra" include("stdlibs/linear_algebra.jl")
+        @safetestset "Random" include("stdlibs/random.jl")
+    end
+
+    @testset "Common Integrations" begin
+        # most integrations are tested in the integration tests except deps that are
+        # very common
+        @safetestset "SpecialFunctions" include("common_integration/special_functions.jl")
+    end
+
+    @testset "Quality Assurance" begin
+        @safetestset "QA" include("core/qa.jl")
+    end
 end
diff --git a/test/integration/linear_algebra.jl b/test/stdlibs/linear_algebra.jl
similarity index 100%
rename from test/integration/linear_algebra.jl
rename to test/stdlibs/linear_algebra.jl
diff --git a/test/integration/random.jl b/test/stdlibs/random.jl
similarity index 100%
rename from test/integration/random.jl
rename to test/stdlibs/random.jl