Move development dependencies into a dependency group (no more extra)

.pre-commit-config.yaml

@@ -54,7 +54,7 @@ repos:
     rev: v6.0.0
     hooks:
       - id: file-contents-sorter
-        files: ^doc/changes/names.inc|^.mailmap
+        files: ^doc/changes/names.inc|^.mailmap|^doc/sphinxext/related_software.txt
         args: ["--ignore-case"]
 
   - repo: https://github.com/pappasam/toml-sort

azure-pipelines.yml

@@ -52,7 +52,7 @@ stages:
           - bash: |
               set -eo pipefail
               python -m pip install --progress-bar off --upgrade pip build
-              python -m pip install --progress-bar off -ve .[hdf5,test]
+              python -m pip install --progress-bar off -ve .[hdf5] --group=test
               python -m pip uninstall -yq pytest-qt  # don't want to set up display, etc. for this
               pre-commit install --install-hooks
             displayName: Install dependencies
@@ -116,7 +116,7 @@ stages:
               python -m pip install --progress-bar off --upgrade pip
               python -m pip install --progress-bar off "mne-qt-browser[opengl] @ git+https://github.com/mne-tools/mne-qt-browser.git" pyvista scikit-learn python-picard qtpy nibabel sphinx-gallery "PySide6!=6.8.0,!=6.8.0.1,!=6.8.1.1,!=6.9.1" pandas neo pymatreader antio defusedxml
               python -m pip uninstall -yq mne
-              python -m pip install --progress-bar off --upgrade -e .[test]
+              python -m pip install --progress-bar off --upgrade -e . --group=test
             displayName: 'Install dependencies with pip'
           - bash: |
               set -e
@@ -173,7 +173,7 @@ stages:
               python -m pip install --progress-bar off --upgrade pip
               python -m pip install --progress-bar off --upgrade --pre --only-binary=\"numpy,scipy,matplotlib,vtk\" numpy scipy matplotlib vtk
               python -c "import vtk"
-              python -m pip install --progress-bar off --upgrade -ve .[full,test_extra]
+              python -m pip install --progress-bar off --upgrade -ve .[full] --group=test_extra
             displayName: 'Install dependencies with pip'
           - bash: |
               set -e

doc/changes/dev/13452.other.rst

@@ -0,0 +1,3 @@
+Removed development dependencies from user-visible "extras"; they're now dependency
+groups only visible to developers (and can be installed for example via
+``pip install --group dev`` with pip version 25.1 or later), by `Richard Höchenberger`_

doc/development/contributing.rst

@@ -304,11 +304,11 @@ be reflected the next time you open a Python interpreter and ``import mne``
 Finally, we'll add a few dependencies that are not needed for running
 MNE-Python, but are needed for locally running our test suite::
 
-    $ pip install -e ".[test]"
+    $ pip install --group=test
 
 And for building our documentation::
 
-    $ pip install -e ".[doc]"
+    $ pip install --group=doc
     $ conda install graphviz
 
 .. note::

doc/sphinxext/related_software.txt

@@ -0,0 +1,37 @@
+# cross-domain-saliency-maps requirements are onerous (torch and tensorflow)
+# so we don't add it here, and install it separately in circleci_dependencies.sh
+alphaCSC
+autoreject
+bycycle
+conpy
+curryreader
+eeg_positions
+emd
+fooof
+meegkit
+meggie
+mne-ari
+mne-bids-pipeline
+mne-faster
+mne-features
+mne-icalabel
+mne-lsl
+mne-microstates
+mne-nirs
+mne-rsa
+mnelab
+neurodsp
+neurokit2
+niseq
+nitime
+pactools
+plotly
+pycrostates
+pyprep
+pyriemann
+python-picard
+sesameeg
+sleepecg
+tensorpac
+wfdb
+yasa

mne/utils/tests/test_config.py

@@ -132,7 +132,11 @@ def test_sys_info_complete():
     sys_info(fid=out, check_version=False, dependencies="developer")
     out = out.getvalue()
     pyproject = tomllib.loads(pyproject.read_text("utf-8"))
-    deps = pyproject["project"]["optional-dependencies"]["test_extra"]
+    deps = [
+        dep
+        for dep in pyproject["dependency-groups"]["test_extra"]
+        if not isinstance(dep, dict)
+    ]
     for dep in deps:
         dep = dep.split("[")[0].split(">")[0].strip()
         assert f" {dep}" in out, f"Missing in dev config: {dep}"

pyproject.toml

@@ -2,6 +2,74 @@
 build-backend = "hatchling.build"
 requires = ["hatch-vcs", "hatchling"]
 
+[dependency-groups]
+dev = ["pip >= 25.1", "rcssmin", {include-group = "doc"}, {include-group = "test_extra"}]
+# Dependencies for building the documentation
+doc = [
+  "graphviz",
+  "intersphinx_registry >= 0.2405.27",
+  "ipython != 8.7.0",  # also in "full-no-qt" and "test"
+  "memory_profiler",
+  "mne-bids",
+  "mne-connectivity",
+  "mne-gui-addons",
+  "neo",
+  "numpydoc",
+  "openneuro-py",
+  "psutil",
+  "pydata_sphinx_theme >= 0.15.2",
+  "pygments >= 2.13",
+  "pytest",
+  "pyxdf",
+  "pyzmq != 24.0.0",
+  "seaborn != 0.11.2",
+  "selenium >= 4.27.1",
+  "sphinx >= 6",
+  "sphinx-design",
+  "sphinx-gallery >= 0.16",
+  "sphinx_copybutton",
+  "sphinxcontrib-bibtex >= 2.5",
+  "sphinxcontrib-towncrier >=0.5.0a0",
+  "sphinxcontrib-youtube",
+]
+test = [
+  "codespell",
+  "flaky",
+  "ipython != 8.7.0",  # for testing notebook backend; also in "full-no-qt" and "doc"
+  "mypy",
+  "numpydoc",
+  "pre-commit",
+  "pytest >= 8.0",
+  "pytest-cov",
+  "pytest-qt",
+  "pytest-timeout",
+  "ruff",
+  "toml-sort",
+  "tomli; python_version<'3.11'",
+  "twine",
+  "vulture",
+  "wheel",
+]
+# Dependencies for being able to run additional tests (rare/CIs/advanced devs)
+# Changes here should be reflected in the mne/utils/config.py dev dependencies section
+test_extra = [
+  "edfio >= 0.4.10",
+  "eeglabio",
+  "imageio >= 2.6.1",
+  "imageio-ffmpeg >= 0.4.1",
+  "jupyter_client",
+  "mne-bids",
+  "nbclient",
+  "nbformat",
+  "neo",
+  "nitime",
+  "pybv",
+  "snirf",
+  "sphinx-gallery",
+  "statsmodels",
+  {include-group = "test"},
+]
+
 [project]
 authors = [
   {email = "[email protected]", name = "Alexandre Gramfort"},
@@ -52,35 +120,6 @@ scripts = {mne = "mne.commands.utils:main"}
 [project.optional-dependencies]
 # Leave this one here for backward-compat
 data = []
-dev = ["mne[doc,test]", "rcssmin"]
-# Dependencies for building the documentation
-doc = [
-  "graphviz",
-  "intersphinx_registry >= 0.2405.27",
-  "ipython != 8.7.0",  # also in "full-no-qt" and "test"
-  "memory_profiler",
-  "mne-bids",
-  "mne-connectivity",
-  "mne-gui-addons",
-  "neo",
-  "numpydoc",
-  "openneuro-py",
-  "psutil",
-  "pydata_sphinx_theme >= 0.15.2",
-  "pygments >= 2.13",
-  "pytest",
-  "pyxdf",
-  "pyzmq != 24.0.0",
-  "seaborn != 0.11.2",
-  "selenium >= 4.27.1",
-  "sphinx >= 6",
-  "sphinx-design",
-  "sphinx-gallery >= 0.16",
-  "sphinx_copybutton",
-  "sphinxcontrib-bibtex >= 2.5",
-  "sphinxcontrib-towncrier >= 0.5.0a0",
-  "sphinxcontrib-youtube",
-]
 full = ["mne[full-no-qt]", "PyQt6 != 6.6.0", "PyQt6-Qt6 != 6.6.0, != 6.7.0"]
 # Dependencies for full MNE-Python functionality (other than raw/epochs export)
 # We first define a variant without any Qt bindings. The "complete" variant, mne[full],
@@ -138,44 +177,6 @@ full-pyqt6 = ["mne[full]"]
 full-pyside6 = ["mne[full-no-qt]", "PySide6 != 6.7.0, != 6.8.0, != 6.8.0.1, != 6.9.1"]
 # Dependencies for MNE-Python functions that use HDF5 I/O
 hdf5 = ["h5io >= 0.2.4", "pymatreader"]
-# Dependencies for running the test infrastructure
-test = [
-  "codespell",
-  "flaky",
-  "ipython != 8.7.0",  # for testing notebook backend; also in "full-no-qt" and "doc"
-  "mypy",
-  "numpydoc",
-  "pre-commit",
-  "pytest >= 8.0",
-  "pytest-cov",
-  "pytest-qt",
-  "pytest-timeout",
-  "ruff",
-  "toml-sort",
-  "tomli; python_version < '3.11'",
-  "twine",
-  "vulture",
-  "wheel",
-]
-# Dependencies for being able to run additional tests (rare/CIs/advanced devs)
-# Changes here should be reflected in the mne/utils/config.py dev dependencies section
-test_extra = [
-  "edfio >= 0.4.10",
-  "eeglabio",
-  "imageio >= 2.6.1",
-  "imageio-ffmpeg >= 0.4.1",
-  "jupyter_client",
-  "mne-bids",
-  "mne[test]",
-  "nbclient",
-  "nbformat",
-  "neo",
-  "nitime",
-  "pybv",
-  "snirf",
-  "sphinx-gallery",
-  "statsmodels",
-]
 
 [project.urls]
 "Bug Tracker" = "https://github.com/mne-tools/mne-python/issues/"

tools/azure_dependencies.sh

@@ -5,10 +5,10 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 STD_ARGS="--progress-bar off --upgrade"
 python -m pip install $STD_ARGS pip setuptools wheel
 if [ "${TEST_MODE}" == "pip" ]; then
-	python -m pip install $STD_ARGS --only-binary="numba,llvmlite,numpy,scipy,vtk,dipy,openmeeg" -e .[test,full]
+	python -m pip install $STD_ARGS --only-binary="numba,llvmlite,numpy,scipy,vtk,dipy,openmeeg" -e .[full] --group=test
 elif [ "${TEST_MODE}" == "pip-pre" ]; then
 	${SCRIPT_DIR}/install_pre_requirements.sh
-	python -m pip install $STD_ARGS --pre -e .[test_extra]
+	python -m pip install $STD_ARGS --pre -e . --group=test_extra
 	echo "##vso[task.setvariable variable=MNE_TEST_ALLOW_SKIP].*(Requires (spm|brainstorm) dataset|Requires MNE-C|CUDA not|Numba not| on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults).*"
 else
 	echo "Unknown run type ${TEST_MODE}"

tools/circleci_dependencies.sh

@@ -1,19 +1,14 @@
 #!/bin/bash -ef
 
-python -m pip install --upgrade "pip!=20.3.0" build
+python -m pip install --upgrade "pip>=25.1" build
 python -m pip install --upgrade --progress-bar off \
+    -ve .[full] \
+    --group=test \
+    --group=doc \
+    -r doc/sphinxext/related_software.txt \
     --only-binary "numpy,dipy,scipy,matplotlib,pandas,statsmodels" \
-    -ve .[full,test,doc] "numpy>=2" \
-    "git+https://github.com/pyvista/pyvista.git" \
-    "git+https://github.com/sphinx-gallery/sphinx-gallery.git" \
     "git+https://github.com/mne-tools/mne-bids.git" \
     "git+https://github.com/mne-tools/mne-qt-browser.git" \
-    \
-    alphaCSC autoreject bycycle conpy emd fooof meggie \
-    mne-ari mne-bids-pipeline mne-faster mne-features \
-    mne-icalabel mne-lsl mne-microstates mne-nirs mne-rsa \
-    neurodsp neurokit2 niseq nitime pactools mnelab \
-    plotly pycrostates pyprep pyriemann python-picard sesameeg \
-    sleepecg tensorpac yasa meegkit eeg_positions wfdb \
-    curryreader
+    "git+https://github.com/pyvista/pyvista.git" \
+    "git+https://github.com/sphinx-gallery/sphinx-gallery.git"
 python -m pip install --upgrade --progress-bar off --no-deps cross-domain-saliency-maps

tools/github_actions_dependencies.sh

@@ -5,7 +5,6 @@ set -eo pipefail
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 STD_ARGS="--progress-bar off --upgrade"
 INSTALL_ARGS="-e"
-INSTALL_KIND="test_extra,hdf5"
 if [ ! -z "$CONDA_ENV" ]; then
 	echo "Uninstalling MNE for CONDA_ENV=${CONDA_ENV}"
 	# This will fail if mne-base is not in the env (like in our minimial/old envs, so ||true them):
@@ -21,19 +20,27 @@ if [ ! -z "$CONDA_ENV" ]; then
 	STD_ARGS="$STD_ARGS https://github.com/pyvista/pyvista/archive/refs/heads/main.zip"
 	# If on minimal or old, just install testing deps
 	if [[ "${CONDA_ENV}" == "environment_"* ]]; then
-		INSTALL_KIND="test"
+		GROUP="test"
+		EXTRAS=""
 		STD_ARGS="--progress-bar off"
+	else
+		GROUP="test_extra"
+		EXTRAS="[hdf5]"
 	fi
 elif [[ "${MNE_CI_KIND}" == "pip" ]]; then
-	INSTALL_KIND="full-pyside6,$INSTALL_KIND"
+	GROUP="test_extra"
+	EXTRAS="[full-pyside6]"
 else
 	test "${MNE_CI_KIND}" == "pip-pre"
 	STD_ARGS="$STD_ARGS --pre"
 	${SCRIPT_DIR}/install_pre_requirements.sh || exit 1
-	INSTALL_KIND="test_extra"
+	GROUP="test_extra"
+	EXTRAS=""
 fi
 echo ""
 
 echo "::group::Installing test dependencies using pip"
-python -m pip install $STD_ARGS $INSTALL_ARGS .[$INSTALL_KIND]
+set -x
+python -m pip install $STD_ARGS $INSTALL_ARGS .$EXTRAS --group=$GROUP
+set +x
 echo "::endgroup::"

tools/hooks/update_environment_file.py

@@ -15,7 +15,7 @@
 
 # Get our "full" dependences from `pyproject.toml`, but actually ignore the
 # "full" section as it's just "full-noqt" plus PyQt6, and for conda we need PySide
-ignore = ("dev", "doc", "test", "test_extra", "full", "full-pyqt6")
+ignore = ("full", "full-pyqt6")
 deps = set(pyproj["project"]["dependencies"])
 for section, section_deps in pyproj["project"]["optional-dependencies"].items():
     if section not in ignore: