CI docbuild: Run containers explicitly with docker run, docker exec

.ci/docker-exec-script.sh

@@ -0,0 +1,14 @@
+#!/bin/sh -x
+if [ $# -lt 3 ]; then
+    echo >&2 "usage: docker-exec-script.sh CONTAINER WORKDIR [VAR=VALUE...] SCRIPT"
+    exit 1
+fi
+CONTAINER=$1
+WORKDIR=$2
+shift 2
+(echo "cd \"$WORKDIR\"";
+ while [ $# -gt 1 ]; do
+     echo "export \"$1\""
+     shift
+ done;
+ cat "$1") | docker exec -i $CONTAINER bash -ex

.github/workflows/build.yml

@@ -96,15 +96,15 @@ jobs:
           SAGE_NUM_THREADS: 2
 
       - name: Build modularized distributions
-        if: always() && steps.worktree.outcome == 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success'
         run: make V=0 tox && make SAGE_CHECK=no pypi-wheels
         working-directory: ./worktree-image
         env:
           MAKE: make -j2 --output-sync=recurse
           SAGE_NUM_THREADS: 2
 
       - name: Static code check with pyright
-        if: always() && steps.worktree.outcome == 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success'
         uses: jakebailey/pyright-action@v1
         with:
           version: 1.1.332
@@ -116,7 +116,7 @@ jobs:
           NODE_OPTIONS: --max-old-space-size=8192
 
       - name: Static code check with pyright (annotated)
-        if: always() && steps.worktree.outcome == 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success'
         uses: jakebailey/pyright-action@v1
         with:
           version: 1.1.332
@@ -130,7 +130,7 @@ jobs:
 
       - name: Clean (fallback to non-incremental)
         id: clean
-        if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
         run: |
           set -ex
           ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status
@@ -143,7 +143,7 @@ jobs:
         # This step is needed because building the modularized distributions installs some optional packages,
         # so the editable install of sagelib needs to build the corresponding optional extension modules.
         id: build
-        if: always() && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success')
+        if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success')
         run: |
           make build
         working-directory: ./worktree-image
@@ -154,7 +154,7 @@ jobs:
       # Testing
 
       - name: Test changed files (sage -t --new)
-        if: always() && steps.build.outcome == 'success'
+        if: (success() || failure()) && steps.build.outcome == 'success'
         run: |
           # We run tests with "sage -t --new"; this only tests the uncommitted changes.
           ./sage -t --new -p2
@@ -164,7 +164,7 @@ jobs:
           SAGE_NUM_THREADS: 2
 
       - name: Test modularized distributions
-        if: always() && steps.build.outcome == 'success'
+        if: (success() || failure()) && steps.build.outcome == 'success'
         run: make V=0 tox && make pypi-wheels-check
         working-directory: ./worktree-image
         env:
@@ -182,14 +182,14 @@ jobs:
           COLUMNS: 120
 
       - name: Test all files (sage -t --all --long)
-        if: always() && steps.build.outcome == 'success'
+        if: (success() || failure()) && steps.build.outcome == 'success'
         run: |
           ../sage -python -m pip install coverage
           ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303
         working-directory: ./worktree-image/src
 
       - name: Prepare coverage results
-        if: always() && steps.build.outcome == 'success'
+        if: (success() || failure()) && steps.build.outcome == 'success'
         run: |
           ./venv/bin/python3 -m coverage combine src/.coverage/
           ./venv/bin/python3 -m coverage xml
@@ -198,7 +198,7 @@ jobs:
         working-directory: ./worktree-image
 
       - name: Upload coverage to codecov
-        if: always() && steps.build.outcome == 'success'
+        if: (success() || failure()) && steps.build.outcome == 'success'
         uses: codecov/codecov-action@v3
         with:
           directory: ./worktree-image/coverage-report

.github/workflows/doc-build-pdf.yml

@@ -21,43 +21,32 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  get_ci_fixes:
+  build-docs-pdf:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        id: checkout
         uses: actions/checkout@v4
+
       - name: Merge CI fixes from sagemath/sage
         run: |
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
-      - name: Store CI fixes in upstream artifact
-        run: |
-          mkdir -p upstream
-          if git format-patch --stdout test_base > ci_fixes.patch; then
-            cp ci_fixes.patch upstream/
-          fi
-      - uses: actions/upload-artifact@v3
-        with:
-          path: upstream
-          name: upstream
 
-  build-docs-pdf:
-    runs-on: ubuntu-latest
-    container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
-    needs: [get_ci_fixes]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - name: Start container
+        run: |
+          docker run --name BUILD -dit \
+            --mount type=bind,src=$(pwd),dst=$(pwd) \
+            --workdir $(pwd) \
+            ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} /bin/sh
 
       - name: Update system packages
         run: |
           export PATH="build/bin:$PATH"
           eval $(sage-print-system-package-command auto update)
           eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
           eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive)
-
+        shell: sh .ci/docker-exec-script.sh BUILD . {0}
 
       - name: Add prebuilt tree as a worktree
         id: worktree
@@ -66,64 +55,46 @@ jobs:
           git config --global user.email "[email protected]"
           git config --global user.name "Build & Test workflow"
           .ci/retrofit-worktree.sh worktree-image /sage
-
-      - name: Download upstream artifact
-        uses: actions/download-artifact@v3
-        with:
-          path: upstream
-          name: upstream
-
-      - name: Apply CI fixes from sagemath/sage
-        # After applying the fixes, make sure all changes are marked as uncommitted changes.
-        run: |
-          if [ -r upstream/ci_fixes.patch ]; then
-            (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
-          fi
+        shell: sh .ci/docker-exec-script.sh BUILD . {0}
 
       - name: Incremental build
         id: incremental
         run: |
+          export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2
           # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
-          ./bootstrap && make build
-        working-directory: ./worktree-image
-        env:
-          MAKE: make -j2 --output-sync=recurse
-          SAGE_NUM_THREADS: 2
+          ./bootstrap && make sagemath_doc_html-build-deps
+        shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0}
 
       - name: Build (fallback to non-incremental)
         id: build
-        if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
+        if: (success() || failure()) && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
         run: |
-          set -ex
-          make sagelib-clean && git clean -fx src/sage && ./config.status && make build
-        working-directory: ./worktree-image
-        env:
-          MAKE: make -j2 --output-sync=recurse
-          SAGE_NUM_THREADS: 2
+          export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2
+          make sagelib-clean && git clean -fx src/sage && ./config.status && make sagemath_doc_html-build-deps
+        shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0}
 
       - name: Build docs (PDF)
         id: docbuild
-        if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
+        if: (success() || failure()) && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
         run: |
-          make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps
-        working-directory: ./worktree-image
-        env:
-          MAKE: make -j2 --output-sync=recurse
-          SAGE_NUM_THREADS: 2
+          export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=2
+          make doc-clean doc-uninstall; make sagemath_doc_pdf-no-deps
+        shell: sh .ci/docker-exec-script.sh BUILD ./worktree-image {0}
 
       - name: Copy docs
         id: copy
-        if: always() && steps.docbuild.outcome == 'success'
+        if: (success() || failure()) && steps.docbuild.outcome == 'success'
         run: |
           # For some reason the deploy step below cannot find /sage/...
           # So copy everything from there to local folder
           mkdir -p ./docs
           cp -r -L /sage/local/share/doc/sage/pdf ./docs
           # Zip everything for increased performance
           zip -r docs-pdf.zip docs
+        shell: sh .ci/docker-exec-script.sh BUILD . {0}
 
       - name: Upload docs
-        if: always() && steps.copy.outcome == 'success'
+        if: (success() || failure()) && steps.copy.outcome == 'success'
         uses: actions/upload-artifact@v3
         with:
           name: docs-pdf