Add # sage_setup: distribution directives to all files

.ci/write-dockerfile.sh

@@ -269,7 +269,7 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-
 $ADD config/config.rpath /new/config/config.rpath
 $ADD src/doc/bootstrap /new/src/doc/bootstrap
 $ADD src/bin /new/src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt /new/src/
+$ADD src/Pipfile.m4 src/pyproject.toml src/requirements.txt.m4 src/VERSION.txt /new/src/
 $ADD m4 /new/m4
 $ADD pkgs /new/pkgs
 $ADD build /new/build

.gitignore

@@ -185,19 +185,6 @@ __pycache__/
 build/temp.*/
 build/bin/sage-build-env-config
 
-# Generated files in build
-build/pkgs/cypari/version_requirements.txt
-build/pkgs/cysignals/version_requirements.txt
-build/pkgs/cython/version_requirements.txt
-build/pkgs/gmpy2/version_requirements.txt
-build/pkgs/jupyter_core/version_requirements.txt
-build/pkgs/memory_allocator/version_requirements.txt
-build/pkgs/numpy/version_requirements.txt
-build/pkgs/pkgconfig/version_requirements.txt
-build/pkgs/pplpy/version_requirements.txt
-build/pkgs/setuptools/version_requirements.txt
-build/pkgs/wheel/version_requirements.txt
-
 # Generated files in the top-level source trees
 /pkgs/*/build
 /pkgs/*/dist

Makefile

@@ -178,17 +178,6 @@ bootstrap-clean:
 	rm -f src/Pipfile
 	rm -f src/requirements.txt
 	rm -f src/setup.cfg
-	rm -f build/pkgs/cypari/version_requirements.txt
-	rm -f build/pkgs/cysignals/version_requirements.txt
-	rm -f build/pkgs/cython/version_requirements.txt
-	rm -f build/pkgs/gmpy2/version_requirements.txt
-	rm -f build/pkgs/jupyter_core/version_requirements.txt
-	rm -f build/pkgs/memory_allocator/version_requirements.txt
-	rm -f build/pkgs/numpy/version_requirements.txt
-	rm -f build/pkgs/pkgconfig/version_requirements.txt
-	rm -f build/pkgs/pplpy/version_requirements.txt
-	rm -f build/pkgs/setuptools/version_requirements.txt
-	rm -f build/pkgs/wheel/version_requirements.txt
 
 # Remove absolutely everything which isn't part of the git repo
 maintainer-clean: distclean bootstrap-clean

bootstrap

@@ -29,21 +29,10 @@ cd "$SAGE_ROOT"
 
 export PATH="$SAGE_ROOT/build/bin:$PATH"
 
-PKG=build/pkgs/configure
 MAKE="${MAKE:-make}"
-CONFVERSION=$(cat $PKG/package-version.txt)
 
 
 bootstrap () {
-    for pkgname in cypari cysignals cython gmpy2 jupyter_core memory_allocator numpy pkgconfig pplpy setuptools wheel; do
-        # Write the version_requirements.txt files for dependencies declared in pyproject.toml
-        target=build/pkgs/${pkgname}/version_requirements.txt
-        if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-            echo "bootstrap:$LINENO: installing '"$target"'"
-        fi
-        echo "# Generated by SAGE_ROOT/bootstrap based on src/pyproject.toml; do not edit directly" > $target
-        sage-get-system-packages install-requires ${pkgname} >> $target
-    done
     for a in m4/sage_spkg_configures.m4 m4/sage_spkg_versions.m4 m4/sage_spkg_versions_toml.m4; do
         if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
             echo "bootstrap:$LINENO: installing '"$a"'"
@@ -54,6 +43,7 @@ bootstrap () {
     for a in m4/sage_spkg_versions.m4 m4/sage_spkg_versions_toml.m4; do
         echo 'changequote(`>>>'"'"', `<<<'"')dnl" >> $a
     done
+    eval $(sage-package properties --format=shell :all:)
     spkg_configures=""
     # initialize SAGE_ENABLE... options for standard packages
     for pkgname in $(sage-package list :standard:); do
@@ -62,75 +52,67 @@ AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [y
     done
     # --enable-SPKG options
     for pkgname in $(sage-package list :optional: :experimental:); do
+        eval DIR=\$path_$pkgname pkgtype=\$type_$pkgname pkgsource=\$source_$pkgname
+        case "$pkgname:$pkgsource" in
+            *:pip)
                 # Issue #29629: Temporary solution for Sage 9.1: Do not provide
                 # --enable-SPKG options for installing pip packages
-                if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
-                    pkgtype="$(cat build/pkgs/$pkgname/type)"
-                    # Issue #29124: Do not provide --enable-_recommended and similar
-                    case "$pkgname" in
-                        _*) ;;
-                        *)  spkg_configures="$spkg_configures
+                ;;
+            _*:*)
+                # Issue #29124: Do not provide --enable-_recommended and similar
+                ;;
+            *:none)
+                # Issue #31163: Just an optional dummy package
+                spkg_configures="$spkg_configures
 AC_SUBST(SAGE_ENABLE_$pkgname, [if_installed])"
-                            if [ -f build/pkgs/$pkgname/spkg-install -o -f build/pkgs/$pkgname/spkg-install.in ]; then
-                                # Issue #31163: Not just an optional dummy package
-                                spkg_configures="$spkg_configures
-SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG.rst | head -n1 2>/dev/null || echo $pkgname)])"
-                            fi
-                            ;;
-                    esac
-                fi
+                ;;
+            *:*)
+                spkg_configures="$spkg_configures
+AC_SUBST(SAGE_ENABLE_$pkgname, [if_installed])"
+                spkg_configures="$spkg_configures
+SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= "$DIR/SPKG.rst" | head -n1 2>/dev/null || echo $pkgname)])"
+                ;;
+        esac
     done
+    cat >> m4/sage_spkg_configures.m4 <<EOF
+SAGE_SPKG_COLLECT_INIT([$(echo $(sage-package list))])
+EOF
     for pkgname in $(sage-package list --has-file spkg-configure.m4); do
-        echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])"
+        eval DIR=\$path_$pkgname
+        echo "m4_sinclude([$DIR/spkg-configure.m4])"
         config="SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
-        if grep -q SAGE_PYTHON_PACKAGE_CHECK build/pkgs/$pkgname/spkg-configure.m4; then
+        if grep -q SAGE_PYTHON_PACKAGE_CHECK "$DIR/spkg-configure.m4"; then
             spkg_configures_python="$spkg_configures_python
 $config"
         else
             spkg_configures="$spkg_configures
 $config"
         fi
     done >> m4/sage_spkg_configures.m4
-    cat >> m4/sage_spkg_configures.m4 <<EOF
-$spkg_configures
-$spkg_configures_python
-EOF
     for pkgname in $(sage-package list); do
-        DIR=build/pkgs/$pkgname
-        pkgtype="$(cat $DIR/type)"
-        if test -f "$DIR/requirements.txt"; then
-            SPKG_SOURCE=pip
-        elif test ! -f "$DIR/checksums.ini"; then
-            if test -f "$DIR/spkg-install" -o -f "$DIR/spkg-install.in"; then
-                SPKG_SOURCE=script
-            else
-                # a dummy package
-                SPKG_SOURCE=none
-            fi
-        else
-            SPKG_SOURCE=normal
-        fi
-        if test -f "$DIR/trees.txt"; then
-            SPKG_TREE_VAR="$(sed "s/#.*//;" "$DIR/trees.txt")"
-        else
-            SPKG_TREE_VAR=SAGE_LOCAL
-            if test -f "$DIR/requirements.txt" -o -f "$DIR/version_requirements.txt"; then
-                # A Python package
-                SPKG_TREE_VAR=SAGE_VENV
-                echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions.m4
-                echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires-toml ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions_toml.m4
-            fi
+        eval DIR=\$path_$pkgname pkgtype=\$type_$pkgname SPKG_SOURCE=\$source_$pkgname SPKG_TREE_VAR=\$trees_$pkgname
+        if test -f "$DIR/requirements.txt" -o -f "$DIR/version_requirements.txt"; then
+            # A Python package
+            echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions.m4
+            INSTALL_REQUIRES_TOML=
+            echo "define(>>>SPKG_INSTALL_REQUIRES_${pkgname}<<<, >>>$(echo $(sage-get-system-packages install-requires-toml ${pkgname}))<<<)dnl" >> m4/sage_spkg_versions_toml.m4
+            echo "m4_define([SPKG_INSTALL_REQUIRES_${pkgname}], [[$(echo $(sage-get-system-packages install-requires-toml ${pkgname} | sed 's/"/\\"/g'))]])dnl" >> m4/sage_spkg_configures.m4
         fi
         spkg_finalizes="$spkg_finalizes
 SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
     done
-    echo "$spkg_finalizes" >> m4/sage_spkg_configures.m4
     for a in m4/sage_spkg_versions.m4 m4/sage_spkg_versions_toml.m4; do
         echo 'changequote(>>>`<<<, >>>'"'"'<<<)dnl' >> $a
     done
+    cat >> m4/sage_spkg_configures.m4 <<EOF
+$spkg_configures
+$spkg_configures_python
+$spkg_finalizes
+EOF
 
     for pkgname in $(sage-package list --has-file bootstrap "$@"); do
-        (cd build/pkgs/$pkgname && if [ -x bootstrap ]; then ./bootstrap; else echo >&2 "bootstrap:$LINENO: Nothing to do for $pkgname"; fi) || exit 1
+        eval DIR=\$path_$pkgname
+        (cd "$DIR" && if [ -x bootstrap ]; then ./bootstrap; else echo >&2 "bootstrap:$LINENO: Nothing to do for $pkgname"; fi) || exit 1
     done
 
     if [ $# != 0 ]; then
@@ -196,10 +178,10 @@ SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
 bootstrap_download () {
     SAGE_DL_LOGLEVEL=""
     [ "${BOOTSTRAP_QUIET}" = "yes" ] && SAGE_DL_LOGLEVEL="--log=WARNING"
-    sage-download-file ${SAGE_DL_LOGLEVEL} configure-$CONFVERSION.tar.gz
 
+    CONFBALL=$(sage-package download $SAGE_DL_LOGLEVEL configure)
     if [ $? -ne 0 ]; then
-        echo >&2 "Error: downloading configure-$CONFVERSION.tar.gz failed"
+        echo >&2 "Error: downloading configure tarball failed"
         exit 1
     fi
 
@@ -241,30 +223,10 @@ save () {
 	src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \
 	src/Pipfile \
 	src/pyproject.toml \
-	src/requirements.txt \
-	src/setup.cfg \
-	build/pkgs/cypari/version_requirements.txt \
-	build/pkgs/cysignals/version_requirements.txt \
-	build/pkgs/cython/version_requirements.txt \
-	build/pkgs/gmpy2/version_requirements.txt \
-	build/pkgs/jupyter_core/version_requirements.txt \
-	build/pkgs/memory_allocator/version_requirements.txt \
-	build/pkgs/numpy/version_requirements.txt \
-	build/pkgs/pkgconfig/version_requirements.txt \
-	build/pkgs/pplpy/version_requirements.txt \
-	build/pkgs/setuptools/version_requirements.txt \
-	build/pkgs/wheel/version_requirements.txt
-
-    # Update version
-    echo "$NEWCONFVERSION" >$PKG/package-version.txt
+	src/requirements.txt
 
-    # Compute checksum
-    if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-	sage-package fix-checksum configure
-    else
-	# Hide the "Updating checksum..." message
-	sage-package fix-checksum configure > /dev/null
-    fi
+    # Update version; this re-computes the checksum
+    sage-package update configure "$NEWCONFVERSION"
 }
 
 
@@ -304,7 +266,6 @@ do
 done
 shift $(($OPTIND - 1))
 export BOOTSTRAP_QUIET
-CONFBALL="upstream/configure-$CONFVERSION.tar.gz"
 
 if [ $DOWNLOAD$SAVE = yesyes ]; then
     echo >&2 "$0: refusing to download and save."

bootstrap-conda

@@ -18,10 +18,11 @@ SAGELIB_PACKAGES=
 SAGELIB_OPTIONAL_PACKAGES=
 DEVELOP_PACKAGES=
 
+eval $(sage-package properties --format=shell :all:)
+
 for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do
-    PKG_SCRIPTS=build/pkgs/$PKG_BASE
+    eval PKG_SCRIPTS=\$path_$PKG_BASE PKG_TYPE=\$type_$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
-    PKG_TYPE=$(cat $PKG_SCRIPTS/type)
     PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE))
     if [ -n "$PKG_SYSTEM_PACKAGES" ]; then
         if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then
@@ -133,12 +134,11 @@ echo >&2 $0:$LINENO: generate conda environment files
         echo >&4 "  - pip:"
         echo >&5 "  - pip:"
         for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file version_requirements.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do
-            PKG_SCRIPTS=build/pkgs/$PKG_BASE
+            eval PKG_SCRIPTS=\$path_$PKG_BASE PKG_TYPE=\$type_$PKG_BASE
             SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt
             if [ ! -f $SYSTEM_PACKAGES_FILE ]; then
                 SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/version_requirements.txt
             fi
-            PKG_TYPE=$(cat $PKG_SCRIPTS/type)
             if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then
                 : # cannot install packages that depend on the Sage library
             else

build/bin/sage-get-system-packages

@@ -15,26 +15,37 @@ fi
 case "$SYSTEM" in
     install-requires)
         # Collect from src/pyproject.toml or from version_requirements.txt (falling back to requirements.txt) and output it in the format
-        # needed by setup.cfg [options] version_requirements=
-        SYSTEM_PACKAGES_FILE_NAMES="version_requirements.txt requirements.txt"
-        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;"
-        FROM_PYPROJECT_TOML=1
+        # needed by setup.cfg [options] install_requires=
+        SYSTEM_PACKAGES_FILE_NAMES="src/pyproject.toml version_requirements.txt requirements.txt"
+        # also normalizes quotes from "" to ''.
+        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/\"/'/g;"
         COLLECT=
         ;;
     install-requires-toml)
         # Collect from src/pyproject.toml or from version_requirements.txt (falling back to requirements.txt) and output it in the format
         # needed by pyproject.toml [build-system] requires=
-        SYSTEM_PACKAGES_FILE_NAMES="version_requirements.txt requirements.txt"
-        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/^/'/;s/$/',/;"
-        FROM_PYPROJECT_TOML=1
+        SYSTEM_PACKAGES_FILE_NAMES="src/pyproject.toml version_requirements.txt requirements.txt"
+        # also normalizes quotes from '' to "".
+        STRIP_COMMENTS="sed s/#.*//;/^[[:space:]]*$/d;s/'/\"/g;s/^/'/;s/$/',/;"
         COLLECT=
         ;;
     pip)
-        SYSTEM_PACKAGES_FILE_NAMES="requirements.txt version_requirements.txt"
+        SYSTEM_PACKAGES_FILE_NAMES="requirements.txt src/pyproject.toml version_requirements.txt"
         STRIP_COMMENTS='sed s/#.*//;s/[[:space:]]//g;'
-        FROM_PYPROJECT_TOML=1
         COLLECT=echo
         ;;
+    versions)
+        # For use in sage-spkg-info
+        SYSTEM_PACKAGES_FILE_NAMES="package-version.txt requirements.txt src/pyproject.toml version_requirements.txt"
+        strip_comments () {
+            echo "$NAME::"
+            echo
+            sed "s/#.*//;/^[[:space:]]*$/d;s/\"/'/g;s/^/    /;" "$@"
+            echo
+        }
+        STRIP_COMMENTS=strip_comments
+        COLLECT=
+        ;;
     *)
         if [ "$SYSTEM" = auto ]; then
             SYSTEM=$(sage-guess-package-system 2>/dev/null)
@@ -45,7 +56,6 @@ case "$SYSTEM" in
         fi
         SYSTEM_PACKAGES_FILE_NAMES="distros/$SYSTEM.txt"
         STRIP_COMMENTS="sed s/#.*//;s/\${PYTHON_MINOR}/${PYTHON_MINOR}/g"
-        FROM_PYPROJECT_TOML=0
         COLLECT=echo
         ;;
 esac
@@ -57,19 +67,10 @@ case "$SPKGS" in
 esac
 
 for PKG_BASE in $SPKGS; do
-    if [ $FROM_PYPROJECT_TOML -eq 1 ]; then
-        if [ -f "$SAGE_ROOT/src/pyproject.toml" ]; then
-            # Extract from the "requires" block in src/pyproject.toml
-            # Packages are in the format "'sage-conf ~= 10.3b3',"
-            PACKAGE_INFO=$(sed -n '/requires *= *\[/,/^\]/s/^ *'\''\('$PKG_BASE'.*\)'\'',/\1/p' "$SAGE_ROOT/src/pyproject.toml")
-            if [ -n "$PACKAGE_INFO" ]; then
-                echo "$PACKAGE_INFO" | ${STRIP_COMMENTS}
-                continue
-            fi
-        fi
-    fi
-
     case "$SYSTEM:$ENABLE_SYSTEM_SITE_PACKAGES" in
+        versions*)
+            # Show everything.
+            ;;
         install-requires*|pip*)
             # This is output for installation of packages into a Python environment.
             # So it's OK to use any Python packages.
@@ -93,14 +94,35 @@ for PKG_BASE in $SPKGS; do
     esac
 
     for NAME in $SYSTEM_PACKAGES_FILE_NAMES; do
-        SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/build/pkgs/$PKG_BASE/$NAME
-        if [ -f $SYSTEM_PACKAGES_FILE ]; then
-            if [ -z "$COLLECT" ]; then
-                ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE
-            else
-                $COLLECT $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)
-            fi
-            break
-        fi
+        case $NAME in
+            *pyproject.toml)
+                SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/$NAME
+                if [ -f "$SYSTEM_PACKAGES_FILE" ]; then
+                    # Extract from src/pyproject.toml:
+                    #  - from the "[build-system] requires" value
+                    #  - from "[project] dependencies",
+                    #  - from "[project.optional-dependencies]"
+                    # Packages are in the format "'sage-conf ~= 10.3b3',"
+                    # Accept dashes and underscores in package names as equivalent:
+                    PKG_REGEX=$(echo $PKG_BASE | sed 's/[-_]/[-_]/g')
+                    PACKAGE_INFO=$(sed -En '/(requires|dependencies|R) *= *\[/,/^\]/s/^ *'\''('$PKG_REGEX'.*)'\'',/\1/p' "$SAGE_ROOT/src/pyproject.toml" | sort -u)
+                    if [ -n "$PACKAGE_INFO" ]; then
+                        echo "$PACKAGE_INFO" | ${STRIP_COMMENTS}
+                        [ $SYSTEM = versions ] || break
+                    fi
+                fi
+                ;;
+            *)
+                SYSTEM_PACKAGES_FILE="$SAGE_ROOT"/build/pkgs/$PKG_BASE/$NAME
+                if [ -f $SYSTEM_PACKAGES_FILE ]; then
+                    if [ -z "$COLLECT" ]; then
+                        ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE
+                    else
+                        $COLLECT $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)
+                    fi
+                    [ $SYSTEM = versions ] || break
+                fi
+                ;;
+        esac
     done
 done