diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh
new file mode 100755
index 00000000000..f926e59059e
--- /dev/null
+++ b/.ci/retrofit-worktree.sh
@@ -0,0 +1,39 @@
+#!/bin/sh
+if [ $# != 2 ]; then
+ echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY"
+ echo >&2 "Ensures that the current working directory is a git repository,"
+ echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME."
+fi
+WORKTREE_NAME="$1"
+WORKTREE_DIRECTORY="$2"
+
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="ci-sage@example.com"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+
+set -ex
+
+# If actions/checkout downloaded our source tree using the GitHub REST API
+# instead of with git (because do not have git installed in our image),
+# we first make the source tree a repo.
+if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
+
+# Tag this state of the source tree "new". This is what we want to build and test.
+git tag -f new
+
+# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage.
+# But $WORKTREE_DIRECTORY is not a git repository.
+# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new".
+# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
+# Then we update worktree and index with "git reset --hard new".
+# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
+# Finally we reset the index to "old". (This keeps all mtimes unchanged.)
+# The changed files now show up as uncommitted changes.
+# The final "git add -N" makes sure that files that were added in "new" do not show
+# as untracked files, which would be removed by "git clean -fx".
+git worktree add --detach $WORKTREE_NAME
+rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/
+rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME
+if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi
+(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 062fa973ac3..42c4496defe 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,7 +15,11 @@ on:
platform:
description: 'Platform'
required: true
- default: 'ubuntu-focal-standard'
+ default: 'ubuntu-focal'
+ packages:
+ description: 'Platform'
+ required: true
+ default: 'standard'
docker_tag:
description: 'Docker tag'
required: true
@@ -27,171 +31,66 @@ concurrency:
cancel-in-progress: true
jobs:
- get_ci_fixes:
+ changed_files:
runs-on: ubuntu-latest
+ name: List changed packages
+ outputs:
+ uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }}
+ build_targets: ${{ steps.build-targets.outputs.build_targets }}
steps:
- - name: Checkout
- id: checkout
- uses: actions/checkout@v4
- - name: Merge CI fixes from sagemath/sage
- run: |
- .ci/merge-fixes.sh
- env:
- GH_TOKEN: ${{ github.token }}
- - name: Store CI fixes in upstream artifact
- run: |
- mkdir -p upstream
- if git format-patch --stdout test_base > ci_fixes.patch; then
- cp ci_fixes.patch upstream/
- fi
- - uses: actions/upload-artifact@v3
+ - uses: actions/checkout@v4
+ - name: Get all packages that have changed
+ id: changed-packages
+ uses: tj-actions/changed-files@v38
with:
- path: upstream
- name: upstream
+ files_yaml: |
+ configures:
+ - 'build/pkgs/*/spkg-configure.m4'
+ pkgs:
+ - 'build/pkgs/**'
+ - 'pkgs/**'
+ - name: Determine targets to build
+ id: build-targets
+ run: |
+ echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
+ echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT
+ cat $GITHUB_OUTPUT
build:
+ uses: ./.github/workflows/docker.yml
+ with:
+ # Build incrementally from published Docker image
+ incremental: true
+ free_disk_space: true
+ from_docker_repository: ghcr.io/sagemath/sage/
+ from_docker_target: "with-targets"
+ from_docker_tag: "dev"
+ docker_targets: "with-targets"
+ targets: "SAGE_CHECK=no build pypi-wheels"
+ tox_system_factors: >-
+ ["${{ github.event.inputs.platform || 'ubuntu-focal' }}"]
+ tox_packages_factors: >-
+ ["${{ github.event.inputs.packages || 'standard' }}"]
+ image_artifact: image
+
+ test-modularized:
runs-on: ubuntu-latest
- container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
- needs: [get_ci_fixes]
+ container: ghcr.io/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-focal' }}-${{ github.event.inputs.packages || 'packages' }}-with-targets:${{ github.sha }}
+ needs: [build]
+ strategy:
+ fail-fast: false
+ matrix:
+ distribution: ${{ fromJson('["sagemath_repl", "sagemath_categories"]') }}
+ env:
+ IMAGE_ARTIFACT_NAME: image-commit-${{ github.sha }}-tox-docker-ubuntu-focal-standard
steps:
- - name: Checkout
- id: checkout
- uses: actions/checkout@v4
-
- - name: Update system packages
- id: prepare
- run: |
- export PATH="build/bin:$PATH"
- eval $(sage-print-system-package-command auto update)
- eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git)
-
- - name: Add prebuilt tree as a worktree
- id: worktree
- run: |
- set -ex
- git config --global user.email "ci-sage@example.com"
- git config --global user.name "Build & Test workflow"
- git config --global --add safe.directory $(pwd)
- # If actions/checkout downloaded our source tree using the GitHub REST API
- # instead of with git (because do not have git installed in our image),
- # we first make the source tree a repo.
- if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
- # Tag this state of the source tree "new". This is what we want to build and test.
- git tag -f new
- # Our container image contains a source tree in /sage with a full build of Sage.
- # But /sage is not a git repository.
- # We make /sage a worktree whose index is at tag "new".
- # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
- # Then we update worktree and index with "git reset --hard new".
- # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
- # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
- # The changed files now show up as uncommitted changes.
- # The final "git add -N" makes sure that files that were added in "new" do not show
- # as untracked files, which would be removed by "git clean -fx".
- git worktree add --detach worktree-image
- rm -rf /sage/.git && mv worktree-image/.git /sage/
- rm -rf worktree-image && ln -s /sage worktree-image
- if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
- (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
-
- - name: Download upstream artifact
+ - name: Download incremental tar archive artifact
uses: actions/download-artifact@v3
with:
- path: upstream
- name: upstream
-
- - name: Apply CI fixes from sagemath/sage
- # After applying the fixes, make sure all changes are marked as uncommitted changes.
+ name: ${{ env.IMAGE_ARTIFACT_NAME }}
+ - name: Update from incremental tar archive
run: |
- if [ -r upstream/ci_fixes.patch ]; then
- (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
- fi
-
- - name: Incremental build, test changed files (sage -t --new)
- id: incremental
+ (cd / && tar xf -) < *-incremental.tar
+ - name: Test
run: |
- # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
- # We run tests with "sage -t --new"; this only tests the uncommitted changes.
- ./bootstrap && make build && ./sage -t --new -p2
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Build and test modularized distributions
- if: always() && steps.worktree.outcome == 'success'
- run: make V=0 tox && make pypi-wheels
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Set up node to install pyright
- if: always() && steps.worktree.outcome == 'success'
- uses: actions/setup-node@v3
- with:
- node-version: '12'
-
- - name: Install pyright
- if: always() && steps.worktree.outcome == 'success'
- # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239
- run: npm install -g pyright@1.1.232
-
- - name: Static code check with pyright
- if: always() && steps.worktree.outcome == 'success'
- run: pyright
- working-directory: ./worktree-image
-
- - name: Clean (fallback to non-incremental)
- id: clean
- if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
- run: |
- set -ex
- ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status
- working-directory: ./worktree-image
- env:
- MAKE: make -j2
- SAGE_NUM_THREADS: 2
-
- - name: Build
- # This step is needed because building the modularized distributions installs some optional packages,
- # so the editable install of sagelib needs to build the corresponding optional extension modules.
- id: build
- if: always() && (steps.incremental.outcome == 'success' || steps.clean.outcome == 'success')
- run: |
- make build
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Pytest
- if: contains(github.ref, 'pytest')
- run: |
- ../sage -python -m pip install coverage pytest-xdist
- ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true
- working-directory: ./worktree-image/src
- env:
- # Increase the length of the lines in the "short summary"
- COLUMNS: 120
-
- - name: Test all files (sage -t --all --long)
- if: always() && steps.build.outcome == 'success'
- run: |
- ../sage -python -m pip install coverage
- ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303
- working-directory: ./worktree-image/src
-
- - name: Prepare coverage results
- if: always() && steps.build.outcome == 'success'
- run: |
- ./venv/bin/python3 -m coverage combine src/.coverage/
- ./venv/bin/python3 -m coverage xml
- find . -name *coverage*
- working-directory: ./worktree-image
-
- - name: Upload coverage to codecov
- if: always() && steps.build.outcome == 'success'
- uses: codecov/codecov-action@v3
- with:
- files: ./worktree-image/coverage.xml
+ make -j2 V=0 tox-ensure && make -j2 ${{ matrix.distribution }}-check"
diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml
index e520fa17104..944af9a72b3 100644
--- a/.github/workflows/ci-conda.yml
+++ b/.github/workflows/ci-conda.yml
@@ -92,7 +92,7 @@ jobs:
run: |
# Use --no-deps and pip check below to verify that all necessary dependencies are installed via conda.
pip install --no-build-isolation --no-deps -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup
- pip install --no-build-isolation --no-deps -v -v -e ./src
+ pip install --no-build-isolation --no-deps --config-settings editable_mode=compat -v -v -e ./src
env:
SAGE_NUM_THREADS: 2
diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml
deleted file mode 100644
index a4aa9ae99c7..00000000000
--- a/.github/workflows/ci-linux-incremental.yml
+++ /dev/null
@@ -1,132 +0,0 @@
-name: CI Linux incremental
-
-## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
-## whenever a GitHub pull request is opened or synchronized in a repository
-## where GitHub Actions are enabled.
-##
-## It builds and checks some sage spkgs as defined in TARGETS.
-##
-## A job succeeds if there is no error.
-##
-## The build is run with "make V=0", so the build logs of individual packages are suppressed.
-##
-## At the end, all package build logs that contain an error are printed out.
-##
-## After all jobs have finished (or are canceled) and a short delay,
-## tar files of all logs are made available as "build artifacts".
-
-on:
- pull_request:
- types:
- # Defaults
- - opened
- - synchronize
- - reopened
- # When a CI label is added
- - labeled
- workflow_dispatch:
-
-concurrency:
- # Cancel previous runs of this workflow for the same branch
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- packages: write
-
-jobs:
-
- changed_files:
- runs-on: ubuntu-latest
- name: List changed packages
- outputs:
- uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }}
- build_targets: ${{ steps.build-targets.outputs.build_targets }}
- steps:
- - uses: actions/checkout@v4
- - name: Get all packages that have changed
- id: changed-packages
- uses: tj-actions/changed-files@v38
- with:
- files_yaml: |
- configures:
- - 'build/pkgs/*/spkg-configure.m4'
- pkgs:
- - 'build/pkgs/**'
- - 'pkgs/**'
- - name: Determine targets to build
- id: build-targets
- run: |
- echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
- echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT
- cat $GITHUB_OUTPUT
-
- test:
- needs: [changed_files]
- if: |
- github.event_name != 'pull_request' ||
- ((github.event.action != 'labeled' &&
- (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
- contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
- (github.event.action == 'labeled' &&
- (github.event.label.name == 'c: packages: optional' ||
- github.event.label.name == 'c: packages: standard')))
- uses: ./.github/workflows/docker.yml
- with:
- # Build incrementally from published Docker image
- incremental: true
- free_disk_space: true
- from_docker_repository: ghcr.io/sagemath/sage/
- from_docker_target: "with-targets"
- from_docker_tag: "dev"
- docker_targets: "with-targets"
- targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
- tox_system_factors: >-
- ["ubuntu-focal",
- "ubuntu-jammy",
- "ubuntu-mantic",
- "debian-bullseye",
- "debian-bookworm",
- "fedora-30",
- "fedora-38",
- "gentoo-python3.11",
- "debian-bullseye-i386"]
- tox_packages_factors: >-
- ["standard",
- "minimal"]
- docker_push_repository: ghcr.io/${{ github.repository }}/
-
- site:
- needs: [changed_files]
- if: |
- github.event_name != 'pull_request' ||
- ((github.event.action != 'labeled' &&
- (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
- contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
- (github.event.action == 'labeled' &&
- (github.event.label.name == 'c: packages: optional' ||
- github.event.label.name == 'c: packages: standard')))
- uses: ./.github/workflows/docker.yml
- with:
- # Build incrementally from published Docker image
- incremental: true
- free_disk_space: true
- from_docker_repository: ghcr.io/sagemath/sage/
- from_docker_target: "with-targets"
- from_docker_tag: "dev"
- docker_targets: "with-targets"
- targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
- # Only test systems with a usable system python (>= 3.9)
- tox_system_factors: >-
- ["ubuntu-jammy",
- "ubuntu-mantic",
- "debian-bullseye",
- "debian-bookworm",
- "fedora-33",
- "fedora-38",
- "gentoo-python3.11",
- "archlinux",
- "debian-bullseye-i386"]
- tox_packages_factors: >-
- ["standard-sitepackages"]
- docker_push_repository: ghcr.io/${{ github.repository }}/
diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml
deleted file mode 100644
index 0413954210b..00000000000
--- a/.github/workflows/doc-build-pdf.yml
+++ /dev/null
@@ -1,151 +0,0 @@
-name: Build documentation (PDF)
-
-on:
- pull_request:
- push:
- workflow_dispatch:
- # Allow to run manually
- inputs:
- platform:
- description: 'Platform'
- required: true
- default: 'ubuntu-focal-standard'
- docker_tag:
- description: 'Docker tag'
- required: true
- default: 'dev'
-
-concurrency:
- # Cancel previous runs of this workflow for the same branch
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- get_ci_fixes:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout
- id: checkout
- uses: actions/checkout@v4
- - name: Merge CI fixes from sagemath/sage
- run: |
- .ci/merge-fixes.sh
- env:
- GH_TOKEN: ${{ github.token }}
- - name: Store CI fixes in upstream artifact
- run: |
- mkdir -p upstream
- if git format-patch --stdout test_base > ci_fixes.patch; then
- cp ci_fixes.patch upstream/
- fi
- - uses: actions/upload-artifact@v3
- with:
- path: upstream
- name: upstream
-
- build-docs-pdf:
- runs-on: ubuntu-latest
- container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}}
- needs: [get_ci_fixes]
- steps:
- - name: Checkout
- uses: actions/checkout@v4
-
- - name: Update system packages
- run: |
- export PATH="build/bin:$PATH"
- eval $(sage-print-system-package-command auto update)
- eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
- eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive)
-
-
- - name: Add prebuilt tree as a worktree
- id: worktree
- run: |
- set -ex
- git config --global user.email "ci-sage@example.com"
- git config --global user.name "Build & Test workflow"
- git config --global --add safe.directory $(pwd)
- # If actions/checkout downloaded our source tree using the GitHub REST API
- # instead of with git (because do not have git installed in our image),
- # we first make the source tree a repo.
- if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
- # Tag this state of the source tree "new". This is what we want to build and test.
- git tag -f new
- # Our container image contains a source tree in /sage with a full build of Sage.
- # But /sage is not a git repository.
- # We make /sage a worktree whose index is at tag "new".
- # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
- # Then we update worktree and index with "git reset --hard new".
- # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
- # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
- # The changed files now show up as uncommitted changes.
- # The final "git add -N" makes sure that files that were added in "new" do not show
- # as untracked files, which would be removed by "git clean -fx".
- git worktree add --detach worktree-image
- rm -rf /sage/.git && mv worktree-image/.git /sage/
- rm -rf worktree-image && ln -s /sage worktree-image
- if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
- (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
- # Keep track of changes to built HTML
- new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
-
- - name: Download upstream artifact
- uses: actions/download-artifact@v3
- with:
- path: upstream
- name: upstream
-
- - name: Apply CI fixes from sagemath/sage
- # After applying the fixes, make sure all changes are marked as uncommitted changes.
- run: |
- if [ -r upstream/ci_fixes.patch ]; then
- (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
- fi
-
- - name: Incremental build
- id: incremental
- run: |
- # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
- ./bootstrap && make build
- working-directory: ./worktree-image
- env:
- MAKE: make -j2
- SAGE_NUM_THREADS: 2
-
- - name: Build (fallback to non-incremental)
- id: build
- if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
- run: |
- set -ex
- make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
- working-directory: ./worktree-image
- env:
- MAKE: make -j2
- SAGE_NUM_THREADS: 2
-
- - name: Build docs (PDF)
- id: docbuild
- if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
- run: make build V=0 && make doc-pdf
- working-directory: ./worktree-image
- env:
- MAKE: make -j2
- SAGE_NUM_THREADS: 2
-
- - name: Copy docs
- if: always() && steps.docbuild.outcome == 'success'
- run: |
- # For some reason the deploy step below cannot find /sage/...
- # So copy everything from there to local folder
- mkdir -p ./docs
- cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs
- # Zip everything for increased performance
- zip -r docs-pdf.zip docs
-
- - name: Upload docs
- if: always() && steps.copy.outcome == 'success'
- uses: actions/upload-artifact@v3
- with:
- name: docs-pdf
- path: docs-pdf.zip
diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml
deleted file mode 100644
index 355e07ab78e..00000000000
--- a/.github/workflows/doc-build.yml
+++ /dev/null
@@ -1,159 +0,0 @@
-name: Build documentation
-
-on:
- pull_request:
- merge_group:
- push:
- branches:
- - master
- - develop
- workflow_dispatch:
- # Allow to run manually
-
-concurrency:
- # Cancel previous runs of this workflow for the same branch
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-jobs:
- get_ci_fixes:
- runs-on: ubuntu-latest
- steps:
- - name: Checkout
- id: checkout
- uses: actions/checkout@v4
- - name: Merge CI fixes from sagemath/sage
- run: |
- .ci/merge-fixes.sh
- env:
- GH_TOKEN: ${{ github.token }}
- - name: Store CI fixes in upstream artifact
- run: |
- mkdir -p upstream
- if git format-patch --stdout test_base > ci_fixes.patch; then
- cp ci_fixes.patch upstream/
- fi
- - uses: actions/upload-artifact@v3
- with:
- path: upstream
- name: upstream
-
- build-docs:
- runs-on: ubuntu-latest
- container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev
- needs: [get_ci_fixes]
- steps:
- - name: Checkout
- uses: actions/checkout@v4
-
- - name: Update system packages
- run: |
- apt-get update && apt-get install -y git zip
-
- - name: Add prebuilt tree as a worktree
- id: worktree
- run: |
- set -ex
- git config --global user.email "ci-sage@example.com"
- git config --global user.name "Build & Test workflow"
- git config --global --add safe.directory $(pwd)
- # If actions/checkout downloaded our source tree using the GitHub REST API
- # instead of with git (because do not have git installed in our image),
- # we first make the source tree a repo.
- if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi
- # Tag this state of the source tree "new". This is what we want to build and test.
- git tag -f new
- # Our container image contains a source tree in /sage with a full build of Sage.
- # But /sage is not a git repository.
- # We make /sage a worktree whose index is at tag "new".
- # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.)
- # Then we update worktree and index with "git reset --hard new".
- # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.)
- # Finally we reset the index to "old". (This keeps all mtimes unchanged.)
- # The changed files now show up as uncommitted changes.
- # The final "git add -N" makes sure that files that were added in "new" do not show
- # as untracked files, which would be removed by "git clean -fx".
- git worktree add --detach worktree-image
- rm -rf /sage/.git && mv worktree-image/.git /sage/
- rm -rf worktree-image && ln -s /sage worktree-image
- if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi
- (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status)
- # Keep track of changes to built HTML
- new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old")
-
- - name: Download upstream artifact
- uses: actions/download-artifact@v3
- with:
- path: upstream
- name: upstream
-
- - name: Apply CI fixes from sagemath/sage
- # After applying the fixes, make sure all changes are marked as uncommitted changes.
- run: |
- if [ -r upstream/ci_fixes.patch ]; then
- (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch
- fi
-
- - name: Incremental build
- id: incremental
- run: |
- # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps.
- ./bootstrap && make build
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Build (fallback to non-incremental)
- id: build
- if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success'
- run: |
- set -ex
- make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Build docs
- id: docbuild
- if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success')
- # Always non-incremental because of the concern that
- # incremental docbuild may introduce broken links (inter-file references) though build succeeds
- run: |
- set -ex
- export SAGE_USE_CDNS=yes
- mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc
- make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage
- mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git
- ./config.status && make doc-html
- working-directory: ./worktree-image
- env:
- MAKE: make -j2 --output-sync=recurse
- SAGE_NUM_THREADS: 2
-
- - name: Copy docs
- id: copy
- if: always() && steps.docbuild.outcome == 'success'
- run: |
- set -ex
- mkdir -p ./docs
- # Create changelog
- echo '## Preview of CHANGES.html'
- (cd /sage/local/share/doc/sage/html/en && git diff --name-only) | tee ./docs/CHANGES.txt
- (cd /sage/local/share/doc/sage/html/en && git diff; rm -rf .git) > ./docs/html.diff
- echo '## Preview of html.diff'; head -n 400 ./docs/html.diff
- (echo 'HTML diff'; sed -E 's,(.*),
\1,' ./docs/CHANGES.txt) > ./docs/CHANGES.html
- # For some reason the deploy step below cannot find /sage/...
- # So copy everything from there to local folder
- # We also need to replace the symlinks because netlify is not following them
- cp -r -L /sage/local/share/doc/sage/html/en/* ./docs
- # Zip everything for increased performance
- zip -r docs.zip docs
-
- - name: Upload docs
- if: always() && steps.copy.outcome == 'success'
- uses: actions/upload-artifact@v3
- with:
- name: docs
- path: docs.zip
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index a3986899a7d..843bf3c1461 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -85,6 +85,12 @@ on:
required: false
type: string
#
+ # Exporting images as artifacts
+ #
+ image_artifact:
+ required: false
+ type: string
+ #
# Incremental builds
#
docker_targets:
@@ -134,6 +140,7 @@ jobs:
env:
TOX_ENV: "docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}${{ inputs.incremental && '-incremental' || '' }}"
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
+ IMAGE_ARTIFACT_NAME: ${{ inputs.image_artifact }}-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
DOCKER_TARGETS: ${{ inputs.docker_targets }}
TARGETS_PRE: ${{ inputs.targets_pre }}
TARGETS: ${{ inputs.targets }}
@@ -244,4 +251,17 @@ jobs:
if [ -f .tox/$TOX_ENV/Dockertags ]; then
cat .tox/$TOX_ENV/Dockertags
fi
- if: always()
+ if: always() && inputs.docker_push_repository
+ - name: Save build as incremental tar archives
+ if: always() && inputs.image_artifact
+ run: |
+ set -x
+ for tag in $(cat .tox/$TOX_ENV/Dockertags); do
+ docker run $tag tar cvf - --listed-incremental=/sage.snar --no-check-device /sage > "$tag-incremental.tar"
+ done
+ - name: Upload incremental tar archives as an artifact
+ if: always() && inputs.image_artifact
+ uses: actions/upload-artifact@v3
+ with:
+ path: "*-incremental.tar"
+ name: ${{ env.IMAGE_ARTIFACT_NAME }}
diff --git a/.upstream.d/10-SAGE_SERVER b/.upstream.d/10-SAGE_SERVER
new file mode 100644
index 00000000000..40ee9e19450
--- /dev/null
+++ b/.upstream.d/10-SAGE_SERVER
@@ -0,0 +1,2 @@
+# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors.
+${SAGE_SERVER}/spkg/upstream/${SPKG}/
diff --git a/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases b/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases
new file mode 100644
index 00000000000..6bccdd7ff0a
--- /dev/null
+++ b/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases
@@ -0,0 +1,3 @@
+# Upstream packages as uploaded as GitHub release assets.
+# This file is automatically updated by the sage-update-version script.
+https://github.com/sagemath/sage/releases/download/10.1/
diff --git a/.upstream.d/30-www.sagemath.org-mirror_list b/.upstream.d/30-www.sagemath.org-mirror_list
new file mode 100644
index 00000000000..951c1df6cfa
--- /dev/null
+++ b/.upstream.d/30-www.sagemath.org-mirror_list
@@ -0,0 +1 @@
+https://www.sagemath.org/mirror_list
diff --git a/build/bin/sage-dist-helpers b/build/bin/sage-dist-helpers
index 67a2201d31f..73119347438 100644
--- a/build/bin/sage-dist-helpers
+++ b/build/bin/sage-dist-helpers
@@ -321,7 +321,15 @@ sdh_store_wheel() {
mkdir -p "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}" && \
$sudo mv "$wheel" "${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/" || \
sdh_die "Error storing $wheel"
- wheel="${SAGE_DESTDIR}${SAGE_SPKG_WHEELS}/${wheel##*/}"
+ wheel="${SAGE_SPKG_WHEELS}/${wheel##*/}"
+ if [ -n "${SAGE_SPKG_SCRIPTS}" -a -n "${PKG_BASE}" ]; then
+ wheel_basename="${wheel##*/}"
+ distname="${wheel_basename%%-*}"
+ # Record name and wheel file location
+ mkdir -p ${SAGE_DESTDIR}${SAGE_SPKG_SCRIPTS}/${PKG_BASE}
+ echo "${distname} @ file://${wheel}" >> ${SAGE_DESTDIR}${SAGE_SPKG_SCRIPTS}/${PKG_BASE}/spkg-requirements.txt
+ fi
+ wheel="${SAGE_DESTDIR}${wheel}"
}
sdh_store_and_pip_install_wheel() {
@@ -373,11 +381,6 @@ sdh_store_and_pip_install_wheel() {
fi
$sudo sage-pip-install $root $pip_options "$wheel" || \
sdh_die "Error installing ${wheel##*/}"
- if [ -n "${SAGE_PKG_DIR}" ]; then
- # Record name of installed distribution name for uninstallation.
- wheel=${wheel##*/}
- echo "${wheel%%-*}" >> ${SAGE_PKG_DIR}/spkg-piprm-requirements.txt
- fi
}
sdh_pip_uninstall() {
diff --git a/build/bin/sage-spkg b/build/bin/sage-spkg
index 9b38ed6afc7..b540c8e866f 100755
--- a/build/bin/sage-spkg
+++ b/build/bin/sage-spkg
@@ -548,7 +548,7 @@ WRAPPED_SCRIPTS="build install check preinst postinst $INSTALLED_SCRIPTS"
# Prepare script for uninstallation of packages that use sdh_pip_install
# or sdh_store_and_pip_install_wheel.
-echo 'sdh_pip_uninstall -r $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-piprm-requirements.txt' > spkg-piprm.in
+echo 'sdh_pip_uninstall -r $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt' > spkg-piprm.in
for script in $WRAPPED_SCRIPTS; do
# 'Installed' scripts are not run immediately out of the package build
@@ -567,24 +567,6 @@ for script in $WRAPPED_SCRIPTS; do
fi
done
-
-# When there is no spkg-install, assume the "spkg" is a tarball not
-# specifically made for Sage. Since we want it to be as easy as
-# possible to install such a package, we "guess" spkg-install.
-if [ ! -f spkg-install ]; then
- echo '#!/usr/bin/env bash' > spkg-install
- if [ -x configure ]; then
- echo './configure --prefix="$SAGE_INST_LOCAL" && make && $SAGE_SUDO make install' >> spkg-install
- elif [ -f setup.py ]; then
- echo 'python setup.py install' >> spkg-install
- else
- echo >&2 "Error: There is no spkg-install script, no setup.py, and no configure"
- echo >&2 "script, so I do not know how to install $PKG_SRC."
- exit 1
- fi
- chmod +x spkg-install
-fi
-
echo "****************************************************"
echo "Host system:"
uname -a
@@ -722,9 +704,7 @@ unset SAGE_DESTDIR_LOCAL
# removed by sage-spkg-uninstall
INSTALLED_SCRIPTS_DEST="$SAGE_SPKG_SCRIPTS/$PKG_BASE"
-if [ -f spkg-piprm-requirements.txt ]; then
- INSTALLED_SCRIPTS="$INSTALLED_SCRIPTS piprm-requirements.txt"
-else
+if [ ! -f $INSTALLED_SCRIPTS_DEST/spkg-requirements.txt ]; then
# No packages to uninstall with pip, so remove the prepared uninstall script
rm -f spkg-piprm spkg-piprm.in
fi
diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh
index ed5d773793c..f9eed46e4fa 100755
--- a/build/bin/write-dockerfile.sh
+++ b/build/bin/write-dockerfile.sh
@@ -52,7 +52,10 @@ case $SYSTEM in
;;
*)
cat < /dev/null; then \
+ (yes | unminimize) || echo "(ignored)"; \
+ rm -f $(command -v unminimize); \
+ fi
EOF
if [ -n "$DIST_UPGRADE" ]; then
cat <> .gitignore && \
+ ./.ci/retrofit-worktree.sh worktree-image /sage); \
+ else \
+ for a in local logs; do \
+ if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \
+ done; \
+ rm -rf /sage; \
+ mv /new /sage; \
+ fi; \
+ else \
+ mv /new /sage; \
+ fi
WORKDIR /sage
-$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./
-$ADD config/config.rpath config/config.rpath
-$ADD src/doc/bootstrap src/doc/bootstrap
-$ADD src/bin src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt src/
-$ADD m4 ./m4
-$ADD pkgs pkgs
-$ADD build ./build
+
ARG BOOTSTRAP=./bootstrap
$RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN
@@ -271,7 +294,17 @@ ENV MAKE="make -j\${NUMPROC}"
ARG USE_MAKEFLAGS="-k V=0"
ENV SAGE_CHECK=warn
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst"
-$ADD src src
+$ADD .gitignore /new
+$ADD src /new/src
+RUN if command -v git; then \
+ rm -f /sage/.git && \
+ cd /new && \
+ ./.ci/retrofit-worktree.sh worktree-pre /sage; \
+ else \
+ rm -rf /sage/src; \
+ mv /new/src /sage/src; \
+ fi
+
ARG TARGETS="build"
$RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN
diff --git a/build/make/Makefile.in b/build/make/Makefile.in
index cc004d08c3c..6031ae7fa2a 100644
--- a/build/make/Makefile.in
+++ b/build/make/Makefile.in
@@ -468,6 +468,8 @@ wheels:
$(MAKE_REC) SAGE_EDITABLE=no SAGE_WHEELS=yes $(WHEEL_PACKAGES)
@echo "Built wheels are in venv/var/lib/sage/wheels/"
+pypi-wheels-check: $(PYPI_WHEEL_PACKAGES:%=%-check)
+
#==============================================================================
# Setting SAGE_CHECK... variables
#==============================================================================
@@ -724,15 +726,21 @@ $(1)-$(4)-no-deps:
echo "$$($(4)_DISABLED_MESSAGE)" 2>&1; \
exit 1; \
elif [ -x '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' ]; then \
+ rm -rf '$$($(4))/var/lib/sage/scripts/$(1)'; \
cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \
. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
. '$$(SAGE_ROOT)/src/bin/sage-env-config' && \
. '$$(SAGE_ROOT)/src/bin/sage-env' && \
. '$$(SAGE_ROOT)/build/bin/sage-build-env-config' && \
. '$$(SAGE_ROOT)/build/bin/sage-build-env' && \
+ PKG_BASE="$(1)" \
+ PKG_VER="$(2)" \
+ PKG_NAME="$(1)-$(2)" \
SAGE_SPKG_WHEELS=$$($(4))/var/lib/sage/wheels \
+ SAGE_SPKG_SCRIPTS=$$($(4))/var/lib/sage/scripts \
SAGE_INST_LOCAL=$$($(4)) \
- sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install' '$$(SAGE_LOGS)/$(1)-$(2).log' && \
+ SAGE_CHECK=$$(SAGE_CHECK_$(1)) \
+ sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-install && if [ $$$$SAGE_CHECK != no -a -x $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check ]; then $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check; fi' '$$(SAGE_LOGS)/$(1)-$(2).log' && \
rm -f "$$($(4))/$(SPKG_INST_RELDIR)/$(1)"-* && \
touch "$$($(4))/$(SPKG_INST_RELDIR)/$(1)-$(2)"; \
else ( \
@@ -752,6 +760,26 @@ $(1)-$(4)-no-deps:
$(1)-no-deps: $(1)-$(4)-no-deps
+$(1)-$(4)-check:
+ $(PLUS)@if [ -x $$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check ]; then \
+ cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \
+ . '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
+ . '$$(SAGE_ROOT)/src/bin/sage-env-config' && \
+ . '$$(SAGE_ROOT)/src/bin/sage-env' && \
+ . '$$(SAGE_ROOT)/build/bin/sage-build-env-config' && \
+ . '$$(SAGE_ROOT)/build/bin/sage-build-env' && \
+ PKG_BASE="$(1)" \
+ PKG_VER="$(2)" \
+ PKG_NAME="$(1)-$(2)" \
+ SAGE_SPKG_WHEELS=$$($(4))/var/lib/sage/wheels \
+ SAGE_SPKG_SCRIPTS=$$($(4))/var/lib/sage/scripts \
+ SAGE_INST_LOCAL=$$($(4)) \
+ SAGE_CHECK=$$(SAGE_CHECK_$(1)) \
+ sage-logger -p '$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-check' '$$(SAGE_LOGS)/$(1)-$(2).log'; \
+ fi
+
+$(1)-check: $(1)-$(4)-check
+
$(1)-$(4)-uninstall:
-$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)' && \
. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
diff --git a/build/pkgs/sage_sws2rst/spkg-check b/build/pkgs/sage_sws2rst/spkg-check
index 667df773214..7372750871e 100755
--- a/build/pkgs/sage_sws2rst/spkg-check
+++ b/build/pkgs/sage_sws2rst/spkg-check
@@ -1,2 +1,11 @@
#! /bin/sh
-cd src && ./check.sh
+cd src
+./check.sh
+if [ $? -ne 0 ]; then
+ if [ "$SAGE_CHECK" = "warn" ]; then
+ echo >&2 "Warning: Failures testing package $PKG_NAME (ignored)"
+ else
+ echo >&2 "Error testing package $PKG_NAME"
+ exit 1
+ fi
+fi
diff --git a/build/pkgs/sage_sws2rst/spkg-install b/build/pkgs/sage_sws2rst/spkg-install
index 9cb71471fd8..a180fb36542 100755
--- a/build/pkgs/sage_sws2rst/spkg-install
+++ b/build/pkgs/sage_sws2rst/spkg-install
@@ -18,13 +18,3 @@ if [ "$SAGE_EDITABLE" = yes ]; then
else
sdh_pip_install .
fi
-cd ..
-# For type=script packages, spkg-check is not run
-case "$SAGE_CHECK" in
- yes)
- ./spkg-check
- ;;
- warn)
- ./spkg-check || echo >&2 "Warning: Failures testing package sage_sws2rst (ignored)"
- ;;
-esac
diff --git a/build/pkgs/sagemath_categories/spkg-check b/build/pkgs/sagemath_categories/spkg-check
new file mode 120000
index 00000000000..91c6b1835fc
--- /dev/null
+++ b/build/pkgs/sagemath_categories/spkg-check
@@ -0,0 +1 @@
+../sagemath_objects/spkg-check
\ No newline at end of file
diff --git a/build/pkgs/sagemath_environment/spkg-check b/build/pkgs/sagemath_environment/spkg-check
new file mode 120000
index 00000000000..91c6b1835fc
--- /dev/null
+++ b/build/pkgs/sagemath_environment/spkg-check
@@ -0,0 +1 @@
+../sagemath_objects/spkg-check
\ No newline at end of file
diff --git a/build/pkgs/sagemath_objects/spkg-check b/build/pkgs/sagemath_objects/spkg-check
new file mode 100755
index 00000000000..87cea0fc38f
--- /dev/null
+++ b/build/pkgs/sagemath_objects/spkg-check
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+cd src
+
+export PIP_NO_INDEX=true
+export PIP_FIND_LINKS="file://$SAGE_SPKG_WHEELS"
+
+export TOX_PARALLEL_NO_SPINNER=1
+wheel="$(sed -n '1s,.*@ file://,,p' $SAGE_SPKG_SCRIPTS/$PKG_BASE/spkg-requirements.txt)"
+echo Running "tox -r -p auto -v --installpkg $wheel"
+tox -r -p auto -v --installpkg "$wheel"
+status=$?
+case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in
+ 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";;
+ 0:*:*) echo "Passed the test suite";;
+ *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;;
+ *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;;
+ *:yes:0) echo "New failures, not in baseline known-test-failures*.json";;
+ *:yes:*) echo "Failures testing the package";;
+esac
+# Show summaries of failures (suppress lines ending with '[failed in baseline]')
+for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do
+ if [ -r "$f" ]; then
+ echo "$f"
+ grep '^sage -t.*#[^]]*$' "$f"
+ fi
+done
+exit $status
diff --git a/build/pkgs/sagemath_objects/spkg-install b/build/pkgs/sagemath_objects/spkg-install
index 472e7f0d4d6..18ceb18724d 100755
--- a/build/pkgs/sagemath_objects/spkg-install
+++ b/build/pkgs/sagemath_objects/spkg-install
@@ -23,26 +23,3 @@ python3 -m build --outdir "$DIST_DIR"/dist . || sdh_die "Failure building sdist
wheel=$(cd "$DIST_DIR" && sdh_store_wheel . >&2 && echo $wheel)
ls -l "$wheel"
-
-if [ "$SAGE_CHECK" != no ]; then
- export TOX_PARALLEL_NO_SPINNER=1
- echo Running "tox -r -p auto -v --installpkg $wheel"
- tox -r -p auto -v --installpkg $wheel
- status=$?
- case $status:$SAGE_CHECK:$([ -r known-test-failures.json ]; echo $?) in
- 0:*:0) echo "Passed the test suite (modulo baseline known-test-failures*.json)";;
- 0:*:*) echo "Passed the test suite";;
- *:warn:0) echo "Warning: New failures (not in baseline known-test-failures*.json (ignored)"; status=0;;
- *:warn:*) echo "Warning: Failures testing the package (ignored)"; status=0;;
- *:yes:0) echo "New failures, not in baseline known-test-failures*.json";;
- *:yes:*) echo "Failures testing the package";;
- esac
- # Show summaries of failures (suppress lines ending with '[failed in baseline]')
- for f in $(pwd)/.tox/sagepython-sagewheels-nopypi-norequirements*/log/*-command*.log; do
- if [ -r "$f" ]; then
- echo "$f"
- grep '^sage -t.*#[^]]*$' "$f"
- fi
- done
- exit $status
-fi
diff --git a/build/pkgs/sagemath_repl/spkg-check b/build/pkgs/sagemath_repl/spkg-check
new file mode 120000
index 00000000000..91c6b1835fc
--- /dev/null
+++ b/build/pkgs/sagemath_repl/spkg-check
@@ -0,0 +1 @@
+../sagemath_objects/spkg-check
\ No newline at end of file
diff --git a/build/pkgs/setuptools/distros/conda.txt b/build/pkgs/setuptools/distros/conda.txt
index 2602d0f6344..49fe098d9e6 100644
--- a/build/pkgs/setuptools/distros/conda.txt
+++ b/build/pkgs/setuptools/distros/conda.txt
@@ -1,4 +1 @@
-# Set this bound until https://github.com/sagemath/sage/issues/34209 adds support for PEP660 editable builds
-# By setting this version bound, we avoid having to include the following in our installation instructions.
-# export SETUPTOOLS_ENABLE_FEATURES=legacy-editable
-"setuptools<64"
+setuptools
diff --git a/build/pkgs/setuptools_wheel/distros b/build/pkgs/setuptools_wheel/distros
new file mode 120000
index 00000000000..b22be5c01a6
--- /dev/null
+++ b/build/pkgs/setuptools_wheel/distros
@@ -0,0 +1 @@
+../setuptools/distros
\ No newline at end of file
diff --git a/build/pkgs/setuptools_wheel/distros/repology.txt b/build/pkgs/setuptools_wheel/distros/repology.txt
deleted file mode 100644
index 845a263c318..00000000000
--- a/build/pkgs/setuptools_wheel/distros/repology.txt
+++ /dev/null
@@ -1 +0,0 @@
-python:setuptools
diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py
index 12868bf6084..a8baa66da2d 100644
--- a/build/sage_bootstrap/download/mirror_list.py
+++ b/build/sage_bootstrap/download/mirror_list.py
@@ -19,7 +19,7 @@
log = logging.getLogger()
from sage_bootstrap.compat import urllib, urlparse
-from sage_bootstrap.env import SAGE_DISTFILES
+from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT
from fcntl import flock, LOCK_SH, LOCK_EX
from errno import ENOLCK
@@ -41,17 +41,66 @@ class MirrorListException(RuntimeError):
pass
-MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list')
+class MirrorList(object):
+ def __init__(self):
+ self.sources = []
+ upstream_d = os.path.join(SAGE_ROOT, '.upstream.d')
+ for fname in sorted(os.listdir(upstream_d)):
+ if '~' in fname or '#' in fname:
+ # Ignore auto-save and backup files
+ continue
+ try:
+ with open(os.path.join(upstream_d, fname), 'r') as f:
+ for line in f:
+ line = line.strip()
+ if line.startswith('#'):
+ continue
+ if not line:
+ continue
+ line = line.replace('${SAGE_ROOT}', SAGE_ROOT)
+ line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES)
+ if '${SAGE_SERVER}' in line:
+ SAGE_SERVER = os.environ.get("SAGE_SERVER", "")
+ if not SAGE_SERVER:
+ continue
+ line = line.replace('${SAGE_SERVER}', SAGE_SERVER)
+ if line.endswith('mirror_list'):
+ cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2])
+ self.sources.append(MirrorList_from_url(line, cache_filename))
+ else:
+ self.sources.append([line])
+ except IOError:
+ # Silently ignore files that do not exist
+ pass
-class MirrorList(object):
+ def __iter__(self):
+ """
+ Iterate through the list of mirrors.
+
+ This is the main entry point into the mirror list. Every
+ script should just use this function to try mirrors in order
+ of preference. This will not just yield the official mirrors,
+ but also urls for packages that are currently being tested.
+ """
+ for source in self.sources:
+ for mirror in source:
+ yield mirror
+
+
+class MirrorList_from_url(object):
- URL = 'http://www.sagemath.org/mirror_list'
MAXAGE = 24*60*60 # seconds
- def __init__(self):
- self.filename = MIRRORLIST_FILENAME
- self.mirrors = None
+ def __init__(self, url, filename):
+ self.url = url
+ self.filename = filename
+ self._mirrors = None
+
+ @property
+ def mirrors(self):
+ if self._mirrors is not None:
+ return self._mirrors
try:
self.mirrorfile = open(self.filename, 'r+t')
@@ -67,8 +116,10 @@ def __init__(self):
# process while we waited for the lock? Check again.
if self._must_refresh():
self._refresh()
- if self.mirrors is None:
- self.mirrors = self._load()
+ if self._mirrors is None:
+ self._mirrors = self._load()
+
+ return self._mirrors
def _load(self, mirror_list=None):
"""
@@ -147,7 +198,7 @@ def _rank_mirrors(self):
log.info('Cannot time mirrors via proxy, using default order')
else:
timed_mirrors.sort()
- self.mirrors = [m[1] for m in timed_mirrors]
+ self._mirrors = [m[1] for m in timed_mirrors]
log.info('Fastest mirror: ' + self.fastest)
def _age(self):
@@ -176,12 +227,12 @@ def _refresh(self):
"""
log.info('Downloading the Sage mirror list')
try:
- with contextlib.closing(urllib.urlopen(self.URL)) as f:
+ with contextlib.closing(urllib.urlopen(self.url)) as f:
mirror_list = f.read().decode("ascii")
except IOError:
log.critical('Downloading the mirror list failed, using cached version')
else:
- self.mirrors = self._load(mirror_list)
+ self._mirrors = self._load(mirror_list)
self._rank_mirrors()
self._save()
@@ -199,9 +250,9 @@ def __iter__(self):
except KeyError:
pass
for mirror in self.mirrors:
- yield mirror
- # If all else fails: Try the packages we host ourselves
- yield 'http://sagepad.org/'
+ if not mirror.endswith('/'):
+ mirror += '/'
+ yield mirror + '/'.join(['spkg', 'upstream', '${SPKG}'])
@property
def fastest(self):
diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py
index 18e3da97af8..e868c5fa51c 100644
--- a/build/sage_bootstrap/tarball.py
+++ b/build/sage_bootstrap/tarball.py
@@ -159,7 +159,10 @@ def download(self, allow_upstream=False):
successful_download = False
log.info('Attempting to download package {0} from mirrors'.format(self.filename))
for mirror in MirrorList():
- url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename])
+ url = mirror.replace('${SPKG}', self.package.name)
+ if not url.endswith('/'):
+ url += '/'
+ url += self.filename
log.info(url)
try:
Download(url, destination).run()
diff --git a/pkgs/sage-conf_conda/setup.py b/pkgs/sage-conf_conda/setup.py
index 9e8ac353f4f..96f163dfe5a 100644
--- a/pkgs/sage-conf_conda/setup.py
+++ b/pkgs/sage-conf_conda/setup.py
@@ -28,24 +28,21 @@ def run(self):
raise SetupError('No conda environment is active. '
'See https://doc.sagemath.org/html/en/installation/conda.html on how to get started.')
- if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
- print(f'Reusing configured SAGE_ROOT={SAGE_ROOT}')
- else:
- cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc"
- cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"'
- cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)'
- print(f"Running {cmd}")
- sys.stdout.flush()
- if os.system(cmd) != 0:
- if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
- print("Warning: A configuration has been written, but the configure script has exited with an error. "
- "Carefully check any messages above before continuing.")
- else:
- print(f"Error: The configure script has failed; this may be caused by missing build prerequisites.")
- sys.stdout.flush()
- PREREQ_SPKG = "_prereq bzip2 xz libffi" # includes python3 SPKG_DEPCHECK packages
- os.system(f'cd {SAGE_ROOT} && export PACKAGES="$(build/bin/sage-get-system-packages conda {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command conda --verbose --sudo install $PACKAGES && echo ""')
- raise SetupError("configure failed")
+ cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc"
+ cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"'
+ cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)'
+ print(f"Running {cmd}")
+ sys.stdout.flush()
+ if os.system(cmd) != 0:
+ if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')):
+ print("Warning: A configuration has been written, but the configure script has exited with an error. "
+ "Carefully check any messages above before continuing.")
+ else:
+ print(f"Error: The configure script has failed; this may be caused by missing build prerequisites.")
+ sys.stdout.flush()
+ PREREQ_SPKG = "_prereq bzip2 xz libffi" # includes python3 SPKG_DEPCHECK packages
+ os.system(f'cd {SAGE_ROOT} && export PACKAGES="$(build/bin/sage-get-system-packages conda {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command conda --verbose --sudo install $PACKAGES && echo ""')
+ raise SetupError("configure failed")
# In this mode, we never run "make".
diff --git a/src/bin/sage-update-version b/src/bin/sage-update-version
index 29d8c794375..3b7358d2929 100755
--- a/src/bin/sage-update-version
+++ b/src/bin/sage-update-version
@@ -87,6 +87,23 @@ EOF
# Create a top-level VERSION.txt file, which some external utilities rely on
echo "$SAGE_VERSION_BANNER" > "$SAGE_ROOT/VERSION.txt"
+# Add version to the front of GitHub release assets URLs.
+SAGE_MINOR_VERSION=${SAGE_VERSION//.alpha*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.beta*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.dev*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.post*/}
+SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.rc*/}
+( echo "https://github.com/sagemath/sage/releases/download/$SAGE_MINOR_VERSION/"
+ sed '/^#/d' "$SAGE_ROOT/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases"
+) | uniq | head -n 3 > "$SAGE_ROOT/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases.tmp"
+( cat < "$SAGE_ROOT/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases"
+rm -f "$SAGE_ROOT/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases.tmp"
+
# Regenerate auto-generated files tarball
"$SAGE_ROOT/bootstrap" -s
@@ -106,6 +123,7 @@ git commit -m "Updated SageMath version to $SAGE_VERSION" -- \
"$SAGE_ROOT/build/pkgs/configure/package-version.txt" \
"$SAGE_ROOT/build/pkgs/*/install-requires.txt" \
"$SAGE_ROOT"/pkgs/*/VERSION.txt \
+ "$SAGE_ROOT/.upstream.d/20-gitproxy.zycloud.tk-sagemath-sage-releases" \
|| die "Error committing to the repository."
git tag -a "$SAGE_VERSION" -m "$SAGE_VERSION_BANNER" \
diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst
index 14fe52ad2e2..9456deff2b0 100644
--- a/src/doc/en/installation/conda.rst
+++ b/src/doc/en/installation/conda.rst
@@ -136,8 +136,9 @@ Here we assume that you are using a git checkout.
A different Python version can be selected by replacing ``3.11`` by ``3.9``
or ``3.10`` in these commands.
- - Install the build prerequisites and the Sage library::
+ - Bootstrap the source tree and install the build prerequisites and the Sage library::
+ $ ./bootstrap
$ pip install --no-build-isolation -v -v --editable ./pkgs/sage-conf_conda ./pkgs/sage-setup
$ pip install --no-build-isolation --config-settings editable_mode=compat -v -v --editable ./src
@@ -160,7 +161,7 @@ suffices to restart Sage.
After editing any Cython files, rebuild the Sage library using::
- $ pip install --no-build-isolation -v -v --editable src
+ $ pip install --no-build-isolation --config-settings editable_mode=compat -v -v --editable src
In order to update the conda environment later, you can run::
diff --git a/src/tox.ini b/src/tox.ini
index 5021d0aa691..3b935f9b9b7 100644
--- a/src/tox.ini
+++ b/src/tox.ini
@@ -78,12 +78,13 @@ commands =
[testenv:pyright]
description =
run the static typing checker pyright
-deps = pyright
+deps =
+ pyright
setenv =
{[sagedirect]setenv}
HOME={envdir}
- # Fix version, see .github/workflows/build.yml
- PYRIGHT_PYTHON_FORCE_VERSION=1.1.232
+ PYRIGHT_PYTHON_FORCE_VERSION=1.1.331
+ PYRIGHT_PYTHON_GLOBAL_NODE=false
allowlist_externals = {[sagedirect]allowlist_externals}
## We run pyright from within the sage-env so that SAGE_LOCAL is available.
## pyright is already configured via SAGE_ROOT/pyrightconfig.json to use our venv.
@@ -92,7 +93,7 @@ allowlist_externals = {[sagedirect]allowlist_externals}
## and may run out of memory. When no files/directories are given, just run it
## on the packages that already have typing annotations.
commands =
- {env:SAGE} -sh -c 'pyright {posargs:{toxinidir}/sage/combinat {toxinidir}/sage/manifolds}'
+ {env:SAGE} -sh -c 'pyright {posargs:}'
[testenv:pycodestyle]
description =
diff --git a/tox.ini b/tox.ini
index 0f4540605e2..eed3d1310a1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -185,6 +185,7 @@ setenv =
# What system packages should be installed. Default: All standard packages with spkg-configure.
SAGE_PACKAGE_LIST_ARGS=--has-file=spkg-configure.m4 :standard:
recommended: EXTRA_SAGE_PACKAGES_3=_recommended $(head -n 1 build/pkgs/_recommended/dependencies)
+ incremental: EXTRA_SAGE_PACKAGES_4=git
develop: EXTRA_SAGE_PACKAGES_4=_develop $(head -n 1 build/pkgs/_develop/dependencies)
minimal: SAGE_PACKAGE_LIST_ARGS=_prereq
maximal: SAGE_PACKAGE_LIST_ARGS=:standard: :optional:
@@ -489,8 +490,9 @@ setenv =
#
docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:}
docker-incremental: FULL_BASE_IMAGE_AND_TAG={env:FROM_DOCKER_REPOSITORY:ghcr.io/sagemath/sage/}sage-$(echo {envname} | sed -E "s/(docker-|-incremental|-sitepackages)//g")-{env:FROM_DOCKER_TARGET:with-targets}:{env:FROM_DOCKER_TAG:dev}
- docker-incremental: SKIP_SYSTEM_PKG_INSTALL=yes
- docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no
+ # Can SKIP_SYSTEM_PKG_INSTALL if the base image already has git
+ docker-incremental-{develop,recommended,maximal}: SKIP_SYSTEM_PKG_INSTALL=yes
+ docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no
#
docker-nobootstrap: BOOTSTRAP=./bootstrap -D
###