Merge pull request #111 from rake-compiler/flavorjones-202312-musl-support

first-class support for musl platforms

Author: flavorjones

.github/workflows/ci.yml

@@ -45,18 +45,26 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - platform: aarch64-linux
-          - platform: arm-linux
+          - platform: aarch64-linux-gnu
+            alias:    aarch64-linux
+          - platform: aarch64-linux-musl
+          - platform: arm-linux-gnu
+            alias:    arm-linux
+          - platform: arm-linux-musl
           - platform: arm64-darwin
           - platform: jruby
           - platform: x64-mingw-ucrt
             static: true
           - platform: x64-mingw32
             static: true
-          - platform: x86-linux
+          - platform: x86-linux-gnu
+            alias:    x86-linux
+          - platform: x86-linux-musl
           - platform: x86-mingw32
           - platform: x86_64-darwin
-          - platform: x86_64-linux
+          - platform: x86_64-linux-gnu
+            alias:    x86_64-linux
+          - platform: x86_64-linux-musl
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -114,23 +122,29 @@ jobs:
           name: gem-${{ matrix.platform }}-static
           path: test/rcd_test/pkg/*-*-*.gem
 
+      - if: matrix.alias
+        name: Build native gem ${{ matrix.alias }}
+        run: |
+          cd test/rcd_test/
+          bundle install
+          bundle exec rake clean clobber
+          bundle exec rake gem:${{ matrix.alias }}
+
+      - if: matrix.alias
+        name: Upload native gem ${{ matrix.alias }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: gem-${{ matrix.alias }}
+          path: test/rcd_test/pkg/*-*-*.gem
+
   test_source_gem:
-    name: test source
+    name: source gem
     needs: build_source_gem
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - ubuntu
-        ruby:
-          - "3.3"
-          - "3.2"
-          - "3.1"
-          - "3.0"
-          - "2.7"
-          - "2.6"
-          - "2.5"
-          - "2.4"
+        os: [ubuntu]
+        ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"]
     runs-on: ${{ matrix.os }}-latest
     steps:
       - uses: actions/checkout@v3
@@ -141,37 +155,117 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: gem-ruby
-      - name: Install source gem
-        run: gem install --local *.gem --verbose
       - name: Test source gem
         run: |
+          gem install --local *.gem --verbose
           cd test/rcd_test/
           bundle install
           ruby -rrcd_test -S rake test
 
-  test_native_gem:
-    name: test native
+  test-x86_64-linux-setup-ruby:
+    name: "${{ matrix.platform }} setup-ruby(${{ matrix.ruby }})"
     needs: build_native_gem
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - macos
-          - ubuntu
-        ruby:
-          - "3.3"
-          - "3.2"
-          - "3.1"
-          - "3.0"
-          - "2.7"
-          - "2.6"
-          - "2.5"
-          - "2.4"
+        platform: [x86_64-linux, x86_64-linux-gnu]
+        ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"]
+        include:
+          # declare rubygems for each ruby version
+          - { ruby: "2.7", rubygems: "3.4.22" }
+          - { ruby: "2.6", rubygems: "3.4.22" }
+          - { ruby: "2.5", rubygems: "3.3.26" }
+          - { ruby: "2.4", rubygems: "3.3.26" }
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Download gem-${{ matrix.platform }}
+        uses: actions/download-artifact@v3
+        with:
+          name: gem-${{ matrix.platform }}
+      - name: Test gem-${{ matrix.platform }}
+        run: |
+          gem update --system ${{ matrix.rubygems }}
+          gem install --local *.gem --verbose
+          cd test/rcd_test/
+          bundle install
+          ruby -rrcd_test -S rake test
+
+  test_architecture_matrix:
+    name: "${{ matrix.platform }} ${{ matrix.ruby }}"
+    needs: build_native_gem
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - aarch64-linux
+          - aarch64-linux-gnu
+          - aarch64-linux-musl
+          - arm-linux
+          - arm-linux-gnu
+          - arm-linux-musl
+          - x86-linux
+          - x86-linux-gnu
+          - x86-linux-musl
+          - x86_64-linux
+          - x86_64-linux-gnu
+          - x86_64-linux-musl
+        ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"]
+        include:
+          # declare rubygems for each ruby version
+          - { ruby: "2.7", rubygems: "3.4.22" }
+          - { ruby: "2.6", rubygems: "3.4.22" }
+          - { ruby: "2.5", rubygems: "3.3.26" }
+          - { ruby: "2.4", rubygems: "3.3.26" }
+          # declare docker image for each platform
+          - { platform: aarch64-linux-musl, docker_tag: "-alpine" }
+          - { platform: arm-linux-musl, docker_tag: "-alpine" }
+          - { platform: x86-linux-musl, docker_tag: "-alpine" }
+          - { platform: x86_64-linux-musl, docker_tag: "-alpine" }
+          # declare docker platform for each platform
+          - { platform: aarch64-linux, docker_platform: "--platform=linux/arm64" }
+          - { platform: aarch64-linux-gnu, docker_platform: "--platform=linux/arm64" }
+          - { platform: aarch64-linux-musl, docker_platform: "--platform=linux/arm64" }
+          - { platform: arm-linux, docker_platform: "--platform=linux/arm/v7" }
+          - { platform: arm-linux-gnu, docker_platform: "--platform=linux/arm/v7" }
+          - { platform: arm-linux-musl, docker_platform: "--platform=linux/arm/v7" }
+          - { platform: x86-linux, docker_platform: "--platform=linux/386" }
+          - { platform: x86-linux-gnu, docker_platform: "--platform=linux/386" }
+          - { platform: x86-linux-musl, docker_platform: "--platform=linux/386" }
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download gem-${{ matrix.platform }}
+        uses: actions/download-artifact@v3
+        with:
+          name: gem-${{ matrix.platform }}
+      - name: Run tests
+        run: |
+          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+          docker run --rm -v $PWD:/work -w /work \
+            ${{ matrix.docker_platform}} ruby:${{ matrix.ruby }}${{ matrix.docker_tag }} \
+            sh -c "
+              gem update --system ${{ matrix.rubygems }} &&
+              gem install --local *.gem --verbose &&
+              cd test/rcd_test/ &&
+              bundle install &&
+              ruby -rrcd_test -S rake test
+            "
+
+  test_the_rest:
+    name: "${{ matrix.platform }} ${{ matrix.ruby }}"
+    needs: build_native_gem
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos]
+        ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"]
         include:
           - os: macos
             platform: x86_64-darwin
-          - os: ubuntu
-            platform: x86_64-linux
           - os: ubuntu
             ruby: jruby
             platform: jruby
@@ -213,16 +307,15 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: gem-${{ matrix.platform }}
-      - name: Install gem-${{ matrix.platform }}
-        run: gem install --local *.gem --verbose
-      - name: Run tests
+      - name: Test gem-${{ matrix.platform }}
         run: |
+          gem install --local *.gem --verbose
           cd test/rcd_test/
           bundle install
           ruby -rrcd_test -S rake test
 
-  test_static_native_gem:
-    name: test static
+  test_windows_static:
+    name: "static ${{ matrix.platform }} ${{ matrix.ruby }}"
     needs: build_native_gem
     strategy:
       fail-fast: false
@@ -266,35 +359,37 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           name: gem-${{ matrix.platform }}-static
-      - name: Install gem-${{ matrix.platform }}-static
-        run: gem install --local *.gem --verbose
-      - name: Run tests
+      - name: Test gem-${{ matrix.platform }}-static
         run: |
+          gem install --local *.gem --verbose
           cd test/rcd_test/
           bundle install
           ruby -rrcd_test -S rake test
 
-  test_native_gem_multiarch:
-    name: ${{ matrix.platform }} on ${{ matrix.from_image }}
+  test_ad_hoc:
+    name: "${{ matrix.platform }} on ${{ matrix.from_image }}"
     needs: build_native_gem
     strategy:
       fail-fast: false
       matrix:
         include:
           - from_image: amd64/centos
-            platform: x86_64-linux
+            platform: x86_64-linux # centos-8 ships ruby 2.5, rubygems won't recognize -gnu suffix
             dockerfile: centos
           - from_image: navikey/raspbian-bullseye
-            platform: arm-linux
+            platform: arm-linux # bullseye ships ruby 2.7, rubygems won't recognize -gnu suffix
             dockerfile: debian
           - from_image: arm64v8/ubuntu
-            platform: aarch64-linux
+            platform: aarch64-linux # arm64v8 ships ruby 3.0, rubygems won't recognize -gnu suffix
             dockerfile: debian
           - from_image: i386/alpine
-            platform: x86-linux
+            platform: x86-linux-musl
             dockerfile: alpine
           - from_image: arm32v6/alpine
-            platform: arm-linux
+            platform: arm-linux-musl
+            dockerfile: alpine
+          - from_image: alpine
+            platform: x86_64-linux-musl
             dockerfile: alpine
     runs-on: ubuntu-latest
     steps:

.github/workflows/publish-images.yml

@@ -13,15 +13,15 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - x86-mingw32
+          - aarch64-linux-gnu
+          - arm-linux-gnu
+          - arm64-darwin
           - x64-mingw-ucrt
           - x64-mingw32
-          - x86-linux
-          - x86_64-linux
+          - x86-linux-gnu
+          - x86-mingw32
           - x86_64-darwin
-          - arm64-darwin
-          - arm-linux
-          - aarch64-linux
+          - x86_64-linux-gnu
           - jruby
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -1,15 +1,18 @@
 /.bundle/
 /.yardoc
-/Dockerfile.mri.aarch64-linux
-/Dockerfile.mri.arm-linux
+/Dockerfile.mri.aarch64-linux-gnu
+/Dockerfile.mri.aarch64-linux-musl
+/Dockerfile.mri.arm-linux-gnu
+/Dockerfile.mri.arm-linux-musl
 /Dockerfile.mri.arm64-darwin
-/Dockerfile.mri.arm64-linux
 /Dockerfile.mri.x64-mingw-ucrt
 /Dockerfile.mri.x64-mingw32
-/Dockerfile.mri.x86-linux
+/Dockerfile.mri.x86-linux-gnu
+/Dockerfile.mri.x86-linux-musl
 /Dockerfile.mri.x86-mingw32
 /Dockerfile.mri.x86_64-darwin
-/Dockerfile.mri.x86_64-linux
+/Dockerfile.mri.x86_64-linux-gnu
+/Dockerfile.mri.x86_64-linux-musl
 /Gemfile.lock
 /_yardoc/
 /cache/

Dockerfile.mri.erb

@@ -1,7 +1,7 @@
 <%
 image = case platform
-  when /x86_64-linux/ then "quay.io/pypa/manylinux2014_x86_64"
-  when /x86-linux/ then "quay.io/pypa/manylinux2014_i686"
+  when /x86_64-linux-gnu/ then "quay.io/pypa/manylinux2014_x86_64"
+  when /x86-linux-gnu/ then "quay.io/pypa/manylinux2014_i686"
   else "ubuntu:20.04"
 end
 manylinux = !!(image =~ /manylinux/)
@@ -64,12 +64,16 @@ COPY --from=larskanis/mingw64-ucrt:20.04 \
     /debs/
 RUN dpkg -i /debs/*.deb
 
+<% elsif platform =~ /linux-musl/ %>
+COPY build/mk_musl_cross.sh /tmp
+RUN /tmp/mk_musl_cross.sh <%= target %>
+
 <% elsif !manylinux %>
 RUN apt-get -y update && \
     apt-get install -y <%
 if platform =~ /darwin/        %> clang python lzma-dev libxml2-dev libssl-dev libc++-10-dev <% end %><%
-if platform =~ /aarch64-linux/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><%
-if platform =~ /arm-linux/     %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><%
+if platform =~ /aarch64-linux-gnu/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><%
+if platform =~ /arm-linux-gnu/     %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><%
 if platform =~ /x86-mingw32/   %> gcc-mingw-w64-i686 g++-mingw-w64-i686 <% end %><%
 if platform =~ /x64-mingw32/   %> gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 <% end %> && \
     rm -rf /var/lib/apt/lists/*
@@ -144,11 +148,9 @@ else
   ]
 end
 
-xrubies_build_plan.each do |xrubies, bootstrap_ruby_version| %>
-ENV XRUBIES <%= xrubies %>
+strip = '-s' if platform !~ /darwin/
 
-<% strip = '-s' if platform !~ /darwin/ %>
-# Build xruby versions, then cleanup all build artifacts
+xrubies_build_plan.each do |xrubies, bootstrap_ruby_version| %>
 RUN bash -c " \
       rbenv shell <%= bootstrap_ruby_version %> && \
       export CPPFLAGS='<%= "-D__USE_MINGW_ANSI_STDIO=1" if platform =~ /x64-mingw-ucrt/ %>' && \
@@ -157,7 +159,7 @@ RUN bash -c " \
       <%= "export LIBS='-l:libssp.a' &&" if platform =~ /mingw/ %> \
       <%= "export CC=#{target}-clang &&" if platform =~ /darwin/ %> \
       export MAKE='make V=1' && \
-      rake-compiler cross-ruby VERSION=$XRUBIES HOST=<%= target %> && \
+      rake-compiler cross-ruby VERSION=<%= xrubies %> HOST=<%= target %> && \
       rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources \
     "
 <% end %>

History.md

@@ -1,6 +1,9 @@
 next / unreleased
 -----------------
 
+* Linux builds and their images are now fully qualified with the libc flavor. So, `x86_64-linux`
+  should now be referred to as `x86_64-linux-gnu`, and the generated files are also named with the
+  libc name.
 * Replace `rvm` with `rbenv` and `ruby-build`
   - `rvm` has been replaced by `rbenv` and `ruby-build`
     - no longer applying sendfile patches to bootstrap rubies