Defer ASV's Environment Management to Nox

[trexfeathers]

📰 Custom Issue

(See /benchmarks for our current implementation).

Replicate the plugin introduced in SciTools/iris-esmf-regrid#76, which re-uses the Nox environment prep to prep ASV's environment. Any changes/improvements to environment management will therefore be shared with no extra work.

If possible this solution should still support ASV's matrix functionality, to allow parallel testing of different dependencies/versions.

Historic commits

Possibly also write a script to allow benchmarking of pre-Nox commits. The below script has been used offline with some success, but might be better to adapt to use lock files instead of conda env update, since that is known to cause timeouts.

Environment management for historic commits

from hashlib import sha256
import json
from pathlib import Path
from subprocess import run, TimeoutExpired
from sys import argv
from tempfile import NamedTemporaryFile


ENV_DIR = Path(argv[1])
REPO_DIR = Path(argv[2])


def main():
    # Locate dependency information.
    # legacy = method for determining Iris requirements pre commit-4fab30a
    #  - benchmark commit ranges straddle this change.
    legacy_req_path = REPO_DIR / "requirements" / "gen_conda_requirements.py"
    legacy_requirements = legacy_req_path.is_file()
    if legacy_requirements:
        req_command = f"python {legacy_req_path} --groups test"
        legacy_req_file = NamedTemporaryFile("r+")
        run(req_command, shell=True, stdout=legacy_req_file)
        env_spec_path = Path(legacy_req_file.name)
        conda_operation = "install --yes"
    else:
        env_spec_path = REPO_DIR / "requirements" / "ci" / "iris.yml"
        conda_operation = "env update"

    # Store dependency cache info for downstream use.
    with env_spec_path.open("rb") as file:
        new_cache = {"env_spec": sha256(file.read()).hexdigest()}

    cache_dir = ENV_DIR / "env_info_cache"
    # Make the directory if it doesn't exist already.
    cache_dir.mkdir(parents=True, exist_ok=True)
    cache_path = cache_dir / "env_info.json"
    cache_path.touch(exist_ok=True)

    # Check if environment already has correct dependencies.
    with cache_path.open("r") as file:
        try:
            env_ok = new_cache == json.load(file)
        except json.JSONDecodeError:
            env_ok = False

    if not env_ok:
        # Install the correct dependencies into the environment.
        conda_command = f"conda {conda_operation} " \
                        f"--prefix '{ENV_DIR}' " \
                        f"--file '{env_spec_path}'"
        try:
            run(conda_command, shell=True, timeout=900)
        except TimeoutExpired:
            raise TimeoutError(f"Conda installation exceeded 15mins - implies "
                               f"a problem. Consider deleting {ENV_DIR}, then "
                               f"restarting benchmarking at this commit.")
        # Record the dependencies that have been installed.
        with cache_path.open("w") as file:
            file.write(json.dumps(new_cache))

    if legacy_requirements:
        legacy_req_file.close()


if __name__ == "__main__":
    main()