diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4f5d8629e7..167b909ddd 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -14,29 +14,34 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - name: Checkout the branches + - name: Fetch the PR base branch too run: | git fetch --depth=1 origin ${{ github.event.pull_request.base.ref }} git branch _base FETCH_HEAD - git fetch --depth=1 origin ${{ github.ref }} - git branch _head FETCH_HEAD + echo PR_BASE_SHA=$(git rev-parse _base) >> $GITHUB_ENV - - name: Setup asv + - name: Install Nox run: | - pip install asv - cd benchmarks - asv machine --yes + pip install nox - - name: Run benchmarks on source and target - run: | - cd benchmarks - asv continuous --factor 1.2 _base _head - - - name: Write a compare file to the output folder - if: ${{ always() }} + - name: Cache .nox and .asv/env directories + id: cache-env-dir + uses: actions/cache@v2 + with: + path: | + .nox + benchmarks/.asv/env + # Make sure GHA never gets an exact cache match by using the unique + # github.sha. This means it will always store this run as a new + # cache (Nox may have made relevant changes during run). Cache + # restoration still succeeds via the partial restore-key match. + key: ${{ runner.os }}-${{ github.sha }} + restore-keys: ${{ runner.os }} + + - name: Run CI benchmarks run: | - cd benchmarks - asv compare -s _base _head > .asv/compare.txt + mkdir --parents benchmarks/.asv + nox --session="benchmarks(ci compare)" | tee benchmarks/.asv/ci_compare.txt - name: Archive asv results if: ${{ always() }} @@ -45,4 +50,4 @@ jobs: name: asv-report path: | benchmarks/.asv/results - benchmarks/.asv/compare.txt \ No newline at end of file + benchmarks/.asv/ci_compare.txt diff --git a/benchmarks/asv.conf.json b/benchmarks/asv.conf.json index 46ee5aa9e9..9ea1cdb101 100644 --- a/benchmarks/asv.conf.json +++ b/benchmarks/asv.conf.json @@ -3,15 +3,18 @@ "project": "scitools-iris", "project_url": "https://github.com/SciTools/iris", "repo": "..", - "environment_type": "conda-lock", + "environment_type": "nox-conda", "show_commit_url": "http://github.com/scitools/iris/commit/", "benchmark_dir": "./benchmarks", "env_dir": ".asv/env", "results_dir": ".asv/results", "html_dir": ".asv/html", - "plugins": [".conda_lock_plugin"], - // this is not an asv standard config entry, just for our plugin - // path to lockfile, relative to project base - "conda_lockfile": "requirements/ci/nox.lock/py38-linux-64.lock" + "plugins": [".nox_asv_plugin"], + // The commit to checkout to first run Nox to set up the environment. + "nox_setup_commit": "HEAD", + // The path of the noxfile's location relative to the project root. + "noxfile_rel_path": "noxfile.py", + // The ``--session`` arg to be used with ``--install-only`` to prep an environment. + "nox_session_name": "tests" } diff --git a/benchmarks/conda_lock_plugin.py b/benchmarks/conda_lock_plugin.py deleted file mode 100644 index 766f5ab2c4..0000000000 --- a/benchmarks/conda_lock_plugin.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the LGPL license. -# See COPYING and COPYING.LESSER in the root of the repository for full -# licensing details. -""" -ASV plug-in providing an alternative ``Environment`` subclass, which uses Nox -for environment management. - -""" -from asv.console import log -from asv.plugins.conda import Conda, _find_conda - - -class CondaLock(Conda): - """ - Create the environment based on a **version-controlled** lockfile. - - Creating the environment instance is deferred until ``build_project`` time, - when the commit hash etc is known and we can access the lock file. - The environment is then overwritten by the specification provided at the - ``config.conda_lockfile`` path. ``conda.conda_lockfile`` must point to - an @EXPLICIT conda manifest, e.g. the output of either the ``conda-lock`` tool, - or ``conda list --explicit``. - """ - - tool_name = "conda-lock" - - def __init__(self, conf, python, requirements): - self._lockfile_path = conf.conda_lockfile - super().__init__(conf, python, requirements) - - def _uninstall_project(self): - installed_hash = self._get_installed_commit_hash() - if installed_hash: - log.info(f"Clearing conda environment for {self.name}") - self._cache._remove_cache_dir(installed_hash) - # we can only run the uninstall command if an environment has already - # been made before, otherwise there is no python to use to uninstall - super()._uninstall_project() - # TODO: we probably want to conda uninstall all the packages too - # something like: - # conda list --no-pip | sed /^#/d | cut -f 1 -d " " | xargs conda uninstall - - def _setup(self): - # create the shell of a conda environment, that includes no packages - log.info("Creating conda environment for {0}".format(self.name)) - self.run_executable( - _find_conda(), ["create", "-y", "-p", self._path, "--force"] - ) - - def _build_project(self, repo, commit_hash, build_dir): - # at "build" time, we build the environment from the provided lockfile - self.run_executable( - _find_conda(), - [ - "install", - "-y", - "-p", - self._path, - "--file", - f"{build_dir}/{self._lockfile_path}", - ], - ) - log.info( - f"Environment {self.name} updated to spec at {commit_hash[:8]}" - ) - log.debug( - self.run_executable(_find_conda(), ["list", "-p", self._path]) - ) - return super()._build_project(repo, commit_hash, build_dir) diff --git a/benchmarks/nox_asv_plugin.py b/benchmarks/nox_asv_plugin.py new file mode 100644 index 0000000000..228a5dc668 --- /dev/null +++ b/benchmarks/nox_asv_plugin.py @@ -0,0 +1,249 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +ASV plug-in providing an alternative ``Environment`` subclass, which uses Nox +for environment management. + +""" +from importlib.util import find_spec +from pathlib import Path +from shutil import copy2, copytree +from tempfile import TemporaryDirectory + +from asv.config import Config +from asv.console import log +from asv.environment import get_env_name +from asv.plugins.conda import Conda, _find_conda +from asv.repo import get_repo, Repo +from asv import util as asv_util + + +class NoxConda(Conda): + """ + Manage a Conda environment using Nox, updating environment at each commit. + + Defers environment management to the project's noxfile, which must be able + to create/update the benchmarking environment using ``nox --install-only``, + with the ``--session`` specified in ``asv.conf.json.nox_session_name``. + + Notes + ----- + If not all benchmarked commits support this use of Nox: the plugin will + need to be modified to prep the environment in other ways. + + """ + + tool_name = "nox-conda" + + @classmethod + def matches(cls, python: str) -> bool: + """Used by ASV to work out if this type of environment can be used.""" + result = find_spec("nox") is not None + if result: + result = super().matches(python) + + if result: + message = ( + f"NOTE: ASV env match check incomplete. Not possible to know " + f"if selected Nox session (asv.conf.json.nox_session_name) is " + f"compatible with ``--python={python}`` until project is " + f"checked out." + ) + log.warning(message) + + return result + + def __init__(self, conf: Config, python: str, requirements: dict) -> None: + """ + Parameters + ---------- + conf: Config instance + + python : str + Version of Python. Must be of the form "MAJOR.MINOR". + + requirements : dict + Dictionary mapping a PyPI package name to a version + identifier string. + + """ + from nox.sessions import _normalize_path + + # Need to checkout the project BEFORE the benchmark run - to access a noxfile. + self.project_temp_checkout = TemporaryDirectory( + prefix="nox_asv_checkout_" + ) + repo = get_repo(conf) + repo.checkout(self.project_temp_checkout.name, conf.nox_setup_commit) + self.noxfile_rel_path = conf.noxfile_rel_path + self.setup_noxfile = ( + Path(self.project_temp_checkout.name) / self.noxfile_rel_path + ) + self.nox_session_name = conf.nox_session_name + + # Some duplication of parent code - need these attributes BEFORE + # running inherited code. + self._python = python + self._requirements = requirements + self._env_dir = conf.env_dir + + # Prepare the actual environment path, to override self._path. + nox_envdir = str(Path(self._env_dir).absolute() / self.hashname) + nox_friendly_name = self._get_nox_session_name(python) + self._nox_path = Path(_normalize_path(nox_envdir, nox_friendly_name)) + + # For storing any extra conda requirements from asv.conf.json. + self._extra_reqs_path = self._nox_path / "asv-extra-reqs.yaml" + + super().__init__(conf, python, requirements) + + @property + def _path(self) -> str: + """ + Using a property to override getting and setting in parent classes - + unable to modify parent classes as this is a plugin. + + """ + return str(self._nox_path) + + @_path.setter + def _path(self, value) -> None: + """Enforce overriding of this variable by disabling modification.""" + pass + + @property + def name(self) -> str: + """Overridden to prevent inclusion of user input requirements.""" + return get_env_name(self.tool_name, self._python, {}) + + def _get_nox_session_name(self, python: str) -> str: + nox_cmd_substring = ( + f"--noxfile={self.setup_noxfile} " + f"--session={self.nox_session_name} " + f"--python={python}" + ) + + list_output = asv_util.check_output( + ["nox", "--list", *nox_cmd_substring.split(" ")], + display_error=False, + dots=False, + ) + list_output = list_output.split("\n") + list_matches = list(filter(lambda s: s.startswith("*"), list_output)) + matches_count = len(list_matches) + + if matches_count == 0: + message = f"No Nox sessions found for: {nox_cmd_substring} ." + log.error(message) + raise RuntimeError(message) + elif matches_count > 1: + message = ( + f"Ambiguous - >1 Nox session found for: {nox_cmd_substring} ." + ) + log.error(message) + raise RuntimeError(message) + else: + line = list_matches[0] + session_name = line.split(" ")[1] + assert isinstance(session_name, str) + return session_name + + def _nox_prep_env(self, setup: bool = False) -> None: + message = f"Running Nox environment update for: {self.name}" + log.info(message) + + build_root_path = Path(self._build_root) + env_path = Path(self._path) + + def copy_asv_files(src_parent: Path, dst_parent: Path) -> None: + """For copying between self._path and a temporary cache.""" + asv_files = list(src_parent.glob("asv*")) + # build_root_path.name usually == "project" . + asv_files += [src_parent / build_root_path.name] + for src_path in asv_files: + dst_path = dst_parent / src_path.name + if not dst_path.exists(): + # Only cache-ing in case Nox has rebuilt the env @ + # self._path. If the dst_path already exists: rebuilding + # hasn't happened. Also a non-issue when copying in the + # reverse direction because the cache dir is temporary. + if src_path.is_dir(): + func = copytree + else: + func = copy2 + func(src_path, dst_path) + + with TemporaryDirectory(prefix="nox_asv_cache_") as asv_cache: + asv_cache_path = Path(asv_cache) + if setup: + noxfile = self.setup_noxfile + else: + # Cache all of ASV's files as Nox may remove and re-build the environment. + copy_asv_files(env_path, asv_cache_path) + # Get location of noxfile in cache. + noxfile_original = ( + build_root_path / self._repo_subdir / self.noxfile_rel_path + ) + noxfile_subpath = noxfile_original.relative_to( + build_root_path.parent + ) + noxfile = asv_cache_path / noxfile_subpath + + nox_cmd = [ + "nox", + f"--noxfile={noxfile}", + # Place the env in the ASV env directory, instead of the default. + f"--envdir={env_path.parent}", + f"--session={self.nox_session_name}", + f"--python={self._python}", + "--install-only", + "--no-error-on-external-run", + "--verbose", + ] + + _ = asv_util.check_output(nox_cmd) + if not env_path.is_dir(): + message = f"Expected Nox environment not found: {env_path}" + log.error(message) + raise RuntimeError(message) + + if not setup: + # Restore ASV's files from the cache (if necessary). + copy_asv_files(asv_cache_path, env_path) + + def _setup(self) -> None: + """Used for initial environment creation - mimics parent method where possible.""" + try: + self.conda = _find_conda() + except IOError as e: + raise asv_util.UserError(str(e)) + if find_spec("nox") is None: + raise asv_util.UserError("Module not found: nox") + + message = f"Creating Nox-Conda environment for {self.name} ." + log.info(message) + + try: + self._nox_prep_env(setup=True) + finally: + # No longer need the setup checkout now that the environment has been built. + self.project_temp_checkout.cleanup() + + conda_args, pip_args = self._get_requirements(self.conda) + if conda_args or pip_args: + message = ( + "Ignoring user input package requirements. Benchmark " + "environment management is exclusively performed by Nox." + ) + log.warning(message) + + def checkout_project(self, repo: Repo, commit_hash: str) -> None: + """Check out the working tree of the project at given commit hash.""" + super().checkout_project(repo, commit_hash) + self._nox_prep_env() + log.info( + f"Environment {self.name} updated to spec at {commit_hash[:8]}" + ) diff --git a/noxfile.py b/noxfile.py index f7cd86e09f..497330de37 100755 --- a/noxfile.py +++ b/noxfile.py @@ -97,7 +97,7 @@ def cache_cartopy(session: nox.sessions.Session) -> None: """ if not CARTOPY_CACHE_DIR.is_dir(): - session.run( + session.run_always( "python", "-c", "import cartopy; cartopy.io.shapereader.natural_earth()", @@ -152,9 +152,9 @@ def prepare_venv(session: nox.sessions.Session) -> None: verbose = "-v" in session.posargs or "--verbose" in session.posargs if verbose: - session.run("conda", "info") - session.run("conda", "list", f"--prefix={venv_dir}") - session.run( + session.run_always("conda", "info") + session.run_always("conda", "list", f"--prefix={venv_dir}") + session.run_always( "conda", "list", f"--prefix={venv_dir}", @@ -278,3 +278,50 @@ def linkcheck(session: nox.sessions.Session): "linkcheck", external=True, ) + + +@nox.session(python=PY_VER[-1], venv_backend="conda") +@nox.parametrize( + ["ci_mode"], + [True, False], + ids=["ci compare", "full"], +) +def benchmarks(session: nox.sessions.Session, ci_mode: bool): + """ + Perform esmf-regrid performance benchmarks (using Airspeed Velocity). + + Parameters + ---------- + session: object + A `nox.sessions.Session` object. + ci_mode: bool + Run a cut-down selection of benchmarks, comparing the current commit to + the last commit for performance regressions. + + Notes + ----- + ASV is set up to use ``nox --session=tests --install-only`` to prepare + the benchmarking environment. This session environment must use a Python + version that is also available for ``--session=tests``. + + """ + session.install("asv", "nox") + session.cd("benchmarks") + # Skip over setup questions for a new machine. + session.run("asv", "machine", "--yes") + + def asv_exec(*sub_args: str) -> None: + run_args = ["asv", *sub_args] + session.run(*run_args) + + if ci_mode: + # If on a PR: compare to the base (target) branch. + # Else: compare to previous commit. + previous_commit = os.environ.get("PR_BASE_SHA", "HEAD^1") + try: + asv_exec("continuous", "--factor=1.2", previous_commit, "HEAD") + finally: + asv_exec("compare", previous_commit, "HEAD") + else: + # f5ceb808 = first commit supporting nox --install-only . + asv_exec("run", "f5ceb808..HEAD")