diff --git a/pytensor/compile/compilelock.py b/pytensor/compile/compilelock.py index 83bf42866d..a1697e43d1 100644 --- a/pytensor/compile/compilelock.py +++ b/pytensor/compile/compilelock.py @@ -8,8 +8,6 @@ from contextlib import contextmanager from pathlib import Path -import filelock - from pytensor.configdefaults import config @@ -35,8 +33,9 @@ def force_unlock(lock_dir: os.PathLike): lock_dir : os.PathLike Path to a directory that was locked with `lock_ctx`. """ + from filelock import FileLock - fl = filelock.FileLock(Path(lock_dir) / ".lock") + fl = FileLock(Path(lock_dir) / ".lock") fl.release(force=True) dir_key = f"{lock_dir}-{os.getpid()}" @@ -62,6 +61,8 @@ def lock_ctx( Timeout in seconds for waiting in lock acquisition. Defaults to `pytensor.config.compile__timeout`. """ + from filelock import FileLock + if lock_dir is None: lock_dir = config.compiledir @@ -73,7 +74,7 @@ def lock_ctx( if dir_key not in local_mem._locks: local_mem._locks[dir_key] = True - fl = filelock.FileLock(Path(lock_dir) / ".lock") + fl = FileLock(Path(lock_dir) / ".lock") fl.acquire(timeout=timeout) try: yield diff --git a/pytensor/configdefaults.py b/pytensor/configdefaults.py index a81fd63905..fcc36f0c6f 100644 --- a/pytensor/configdefaults.py +++ b/pytensor/configdefaults.py @@ -3,11 +3,10 @@ import os import platform import re -import shutil -import socket import sys import textwrap from pathlib import Path +from shutil import which import numpy as np @@ -349,7 +348,7 @@ def add_compile_configvars(): # Try to find the full compiler path from the name if param != "": - newp = shutil.which(param) + newp = which(param) if newp is not None: param = newp del newp @@ -1190,7 +1189,7 @@ def _get_home_dir() -> Path: "pytensor_version": pytensor.__version__, "numpy_version": np.__version__, "gxx_version": "xxx", - "hostname": socket.gethostname(), + "hostname": platform.node(), } diff --git a/pytensor/configparser.py b/pytensor/configparser.py index 8c6da4a144..4f71e85240 100644 --- a/pytensor/configparser.py +++ b/pytensor/configparser.py @@ -1,6 +1,5 @@ import logging import os -import shlex import sys import warnings from collections.abc import Callable, Sequence @@ -14,6 +13,7 @@ from functools import wraps from io import StringIO from pathlib import Path +from shlex import shlex from pytensor.utils import hash_from_code @@ -541,7 +541,7 @@ def parse_config_string( Parses a config string (comma-separated key=value components) into a dict. """ config_dict = {} - my_splitter = shlex.shlex(config_string, posix=True) + my_splitter = shlex(config_string, posix=True) my_splitter.whitespace = "," my_splitter.whitespace_split = True for kv_pair in my_splitter: diff --git a/pytensor/d3viz/formatting.py b/pytensor/d3viz/formatting.py index b9fb8ee5a5..df39335c19 100644 --- a/pytensor/d3viz/formatting.py +++ b/pytensor/d3viz/formatting.py @@ -12,13 +12,7 @@ from pytensor.compile import Function, builders from pytensor.graph.basic import Apply, Constant, Variable, graph_inputs from pytensor.graph.fg import FunctionGraph -from pytensor.printing import pydot_imported, pydot_imported_msg - - -try: - from pytensor.printing import pd -except ImportError: - pass +from pytensor.printing import _try_pydot_import class PyDotFormatter: @@ -41,8 +35,7 @@ class PyDotFormatter: def __init__(self, compact=True): """Construct PyDotFormatter object.""" - if not pydot_imported: - raise ImportError("Failed to import pydot. " + pydot_imported_msg) + _try_pydot_import() self.compact = compact self.node_colors = { @@ -115,6 +108,8 @@ def __call__(self, fct, graph=None): pydot.Dot Pydot graph of `fct` """ + pd = _try_pydot_import() + if graph is None: graph = pd.Dot() @@ -356,6 +351,8 @@ def type_to_str(t): def dict_to_pdnode(d): """Create pydot node from dict.""" + pd = _try_pydot_import() + e = dict() for k, v in d.items(): if v is not None: diff --git a/pytensor/graph/rewriting/basic.py b/pytensor/graph/rewriting/basic.py index faec736c98..344d6a1940 100644 --- a/pytensor/graph/rewriting/basic.py +++ b/pytensor/graph/rewriting/basic.py @@ -5,7 +5,6 @@ import functools import inspect import logging -import pdb import sys import time import traceback @@ -237,6 +236,8 @@ def warn(cls, exc, self, rewriter): if config.on_opt_error == "raise": raise exc elif config.on_opt_error == "pdb": + import pdb + pdb.post_mortem(sys.exc_info()[2]) def __init__(self, *rewrites, failure_callback=None): @@ -1752,6 +1753,8 @@ def warn(cls, exc, nav, repl_pairs, node_rewriter, node): _logger.error("TRACEBACK:") _logger.error(traceback.format_exc()) if config.on_opt_error == "pdb": + import pdb + pdb.post_mortem(sys.exc_info()[2]) elif isinstance(exc, AssertionError) or config.on_opt_error == "raise": # We always crash on AssertionError because something may be diff --git a/pytensor/link/c/cmodule.py b/pytensor/link/c/cmodule.py index f1f098edbf..c992d0506e 100644 --- a/pytensor/link/c/cmodule.py +++ b/pytensor/link/c/cmodule.py @@ -26,19 +26,12 @@ from typing import TYPE_CHECKING, Protocol, cast import numpy as np -from setuptools._distutils.sysconfig import ( - get_config_h_filename, - get_config_var, - get_python_inc, - get_python_lib, -) # we will abuse the lockfile mechanism when reading and writing the registry from pytensor.compile.compilelock import lock_ctx from pytensor.configdefaults import config, gcc_version_str from pytensor.configparser import BoolParam, StrParam from pytensor.graph.op import Op -from pytensor.link.c.exceptions import CompileError, MissingGXX from pytensor.utils import ( LOCAL_BITWIDTH, flatten, @@ -266,6 +259,8 @@ def list_code(self, ofile=sys.stdout): def _get_ext_suffix(): """Get the suffix for compiled extensions""" + from setuptools._distutils.sysconfig import get_config_var + dist_suffix = get_config_var("EXT_SUFFIX") if dist_suffix is None: dist_suffix = get_config_var("SO") @@ -1697,6 +1692,8 @@ def get_gcc_shared_library_arg(): def std_include_dirs(): + from setuptools._distutils.sysconfig import get_python_inc + numpy_inc_dirs = [np.get_include()] py_inc = get_python_inc() py_plat_spec_inc = get_python_inc(plat_specific=True) @@ -1709,6 +1706,12 @@ def std_include_dirs(): @is_StdLibDirsAndLibsType def std_lib_dirs_and_libs() -> tuple[list[str], ...] | None: + from setuptools._distutils.sysconfig import ( + get_config_var, + get_python_inc, + get_python_lib, + ) + # We cache the results as on Windows, this trigger file access and # this method is called many times. if std_lib_dirs_and_libs.data is not None: @@ -2388,23 +2391,6 @@ def join_options(init_part): # xcode's version. cxxflags.append("-ld64") - if sys.platform == "win32": - # Workaround for https://github.com/Theano/Theano/issues/4926. - # https://github.com/python/cpython/pull/11283/ removed the "hypot" - # redefinition for recent CPython versions (>=2.7.16 and >=3.7.3). - # The following nullifies that redefinition, if it is found. - python_version = sys.version_info[:3] - if (3,) <= python_version < (3, 7, 3): - config_h_filename = get_config_h_filename() - try: - with open(config_h_filename) as config_h: - if any( - line.startswith("#define hypot _hypot") for line in config_h - ): - cxxflags.append("-D_hypot=hypot") - except OSError: - pass - return cxxflags @classmethod @@ -2555,8 +2541,9 @@ def compile_str( """ # TODO: Do not do the dlimport in this function - if not config.cxx: + from pytensor.link.c.exceptions import MissingGXX + raise MissingGXX("g++ not available! We can't compile c code.") if include_dirs is None: @@ -2586,6 +2573,8 @@ def compile_str( cppfile.write("\n") if platform.python_implementation() == "PyPy": + from setuptools._distutils.sysconfig import get_config_var + suffix = "." + get_lib_extension() dist_suffix = get_config_var("SO") @@ -2642,6 +2631,8 @@ def print_command_line_error(): status = p_out[2] if status: + from pytensor.link.c.exceptions import CompileError + tf = tempfile.NamedTemporaryFile( mode="w", prefix="pytensor_compilation_error_", delete=False ) diff --git a/pytensor/link/vm.py b/pytensor/link/vm.py index a9d625a8da..af44af3254 100644 --- a/pytensor/link/vm.py +++ b/pytensor/link/vm.py @@ -19,7 +19,6 @@ from pytensor.configdefaults import config from pytensor.graph.basic import Apply, Constant, Variable from pytensor.link.basic import Container, LocalLinker -from pytensor.link.c.exceptions import MissingGXX from pytensor.link.utils import ( gc_helper, get_destroy_dependencies, @@ -1006,6 +1005,8 @@ def make_vm( compute_map, updated_vars, ): + from pytensor.link.c.exceptions import MissingGXX + pre_call_clear = [storage_map[v] for v in self.no_recycling] try: diff --git a/pytensor/printing.py b/pytensor/printing.py index 9a34317c40..6a18f6e8e5 100644 --- a/pytensor/printing.py +++ b/pytensor/printing.py @@ -26,39 +26,6 @@ IDTypesType = Literal["id", "int", "CHAR", "auto", ""] -pydot_imported = False -pydot_imported_msg = "" -try: - # pydot-ng is a fork of pydot that is better maintained - import pydot_ng as pd - - if pd.find_graphviz(): - pydot_imported = True - else: - pydot_imported_msg = "pydot-ng can't find graphviz. Install graphviz." -except ImportError: - try: - # fall back on pydot if necessary - import pydot as pd - - if hasattr(pd, "find_graphviz"): - if pd.find_graphviz(): - pydot_imported = True - else: - pydot_imported_msg = "pydot can't find graphviz" - else: - pd.Dot.create(pd.Dot()) - pydot_imported = True - except ImportError: - # tests should not fail on optional dependency - pydot_imported_msg = ( - "Install the python package pydot or pydot-ng. Install graphviz." - ) - except Exception as e: - pydot_imported_msg = "An error happened while importing/trying pydot: " - pydot_imported_msg += str(e.args) - - _logger = logging.getLogger("pytensor.printing") VALID_ASSOC = {"left", "right", "either"} @@ -1196,6 +1163,48 @@ def __call__(self, *args): } +def _try_pydot_import(): + pydot_imported = False + pydot_imported_msg = "" + try: + # pydot-ng is a fork of pydot that is better maintained + import pydot_ng as pd + + if pd.find_graphviz(): + pydot_imported = True + else: + pydot_imported_msg = "pydot-ng can't find graphviz. Install graphviz." + except ImportError: + try: + # fall back on pydot if necessary + import pydot as pd + + if hasattr(pd, "find_graphviz"): + if pd.find_graphviz(): + pydot_imported = True + else: + pydot_imported_msg = "pydot can't find graphviz" + else: + pd.Dot.create(pd.Dot()) + pydot_imported = True + except ImportError: + # tests should not fail on optional dependency + pydot_imported_msg = ( + "Install the python package pydot or pydot-ng. Install graphviz." + ) + except Exception as e: + pydot_imported_msg = "An error happened while importing/trying pydot: " + pydot_imported_msg += str(e.args) + + if not pydot_imported: + raise ImportError( + "Failed to import pydot. You must install graphviz " + "and either pydot or pydot-ng for " + f"`pydotprint` to work:\n {pydot_imported_msg}", + ) + return pd + + def pydotprint( fct, outfile: Path | str | None = None, @@ -1288,6 +1297,8 @@ def pydotprint( scan separately after the top level debugprint output. """ + pd = _try_pydot_import() + from pytensor.scan.op import Scan if colorCodes is None: @@ -1320,12 +1331,6 @@ def pydotprint( outputs = fct.outputs topo = fct.toposort() fgraph = fct - if not pydot_imported: - raise RuntimeError( - "Failed to import pydot. You must install graphviz " - "and either pydot or pydot-ng for " - f"`pydotprint` to work:\n {pydot_imported_msg}", - ) g = pd.Dot() diff --git a/pytensor/scan/op.py b/pytensor/scan/op.py index bfe04a94d7..a01347ef9c 100644 --- a/pytensor/scan/op.py +++ b/pytensor/scan/op.py @@ -74,7 +74,6 @@ from pytensor.graph.replace import clone_replace from pytensor.graph.utils import InconsistencyError, MissingInputError from pytensor.link.c.basic import CLinker -from pytensor.link.c.exceptions import MissingGXX from pytensor.printing import op_debug_information from pytensor.scan.utils import ScanProfileStats, Validator, forced_replace, safe_new from pytensor.tensor.basic import as_tensor_variable @@ -1499,6 +1498,7 @@ def make_thunk(self, node, storage_map, compute_map, no_recycling, impl=None): then it must not do so for variables in the no_recycling list. """ + from pytensor.link.c.exceptions import MissingGXX # Before building the thunk, validate that the inner graph is # coherent diff --git a/pytensor/tensor/blas.py b/pytensor/tensor/blas.py index 6170a02a98..3c38a9c501 100644 --- a/pytensor/tensor/blas.py +++ b/pytensor/tensor/blas.py @@ -111,50 +111,19 @@ _logger = logging.getLogger("pytensor.tensor.blas") -try: - import scipy.linalg.blas - - have_fblas = True - try: - fblas = scipy.linalg.blas.fblas - except AttributeError: - # A change merged in Scipy development version on 2012-12-02 replaced - # `scipy.linalg.blas.fblas` with `scipy.linalg.blas`. - # See http://github.com/scipy/scipy/pull/358 - fblas = scipy.linalg.blas - _blas_gemv_fns = { - np.dtype("float32"): fblas.sgemv, - np.dtype("float64"): fblas.dgemv, - np.dtype("complex64"): fblas.cgemv, - np.dtype("complex128"): fblas.zgemv, - } -except ImportError as e: - have_fblas = False - # This is used in Gemv and ScipyGer. We use CGemv and CGer - # when config.blas__ldflags is defined. So we don't need a - # warning in that case. - if not config.blas__ldflags: - _logger.warning( - "Failed to import scipy.linalg.blas, and " - "PyTensor flag blas__ldflags is empty. " - "Falling back on slower implementations for " - "dot(matrix, vector), dot(vector, matrix) and " - f"dot(vector, vector) ({e!s})" - ) - # If check_init_y() == True we need to initialize y when beta == 0. def check_init_y(): + # TODO: What is going on here? + from scipy.linalg.blas import get_blas_funcs + if check_init_y._result is None: - if not have_fblas: # pragma: no cover - check_init_y._result = False - else: - y = float("NaN") * np.ones((2,)) - x = np.ones((2,)) - A = np.ones((2, 2)) - gemv = _blas_gemv_fns[y.dtype] - gemv(1.0, A.T, x, 0.0, y, overwrite_y=True, trans=True) - check_init_y._result = np.isnan(y).any() + y = float("NaN") * np.ones((2,)) + x = np.ones((2,)) + A = np.ones((2, 2)) + gemv = get_blas_funcs("gemv", dtype=y.dtype) + gemv(1.0, A.T, x, 0.0, y, overwrite_y=True, trans=True) + check_init_y._result = np.isnan(y).any() return check_init_y._result @@ -211,14 +180,15 @@ def make_node(self, y, alpha, A, x, beta): return Apply(self, inputs, [y.type()]) def perform(self, node, inputs, out_storage): + from scipy.linalg.blas import get_blas_funcs + y, alpha, A, x, beta = inputs if ( - have_fblas - and y.shape[0] != 0 + y.shape[0] != 0 and x.shape[0] != 0 - and y.dtype in _blas_gemv_fns + and y.dtype in {"float32", "float64", "complex64", "complex128"} ): - gemv = _blas_gemv_fns[y.dtype] + gemv = get_blas_funcs("gemv", dtype=y.dtype) if A.shape[0] != y.shape[0] or A.shape[1] != x.shape[0]: raise ValueError( diff --git a/pytensor/tensor/blas_scipy.py b/pytensor/tensor/blas_scipy.py index 16fb90988b..bb3ccf9354 100644 --- a/pytensor/tensor/blas_scipy.py +++ b/pytensor/tensor/blas_scipy.py @@ -2,30 +2,19 @@ Implementations of BLAS Ops based on scipy's BLAS bindings. """ -import numpy as np - -from pytensor.tensor.blas import Ger, have_fblas - - -if have_fblas: - from pytensor.tensor.blas import fblas - - _blas_ger_fns = { - np.dtype("float32"): fblas.sger, - np.dtype("float64"): fblas.dger, - np.dtype("complex64"): fblas.cgeru, - np.dtype("complex128"): fblas.zgeru, - } +from pytensor.tensor.blas import Ger class ScipyGer(Ger): def perform(self, node, inputs, output_storage): + from scipy.linalg.blas import get_blas_funcs + cA, calpha, cx, cy = inputs (cZ,) = output_storage # N.B. some versions of scipy (e.g. mine) don't actually work # in-place on a, even when I tell it to. A = cA - local_ger = _blas_ger_fns[cA.dtype] + local_ger = get_blas_funcs("ger", dtype=cA.dtype) if A.size == 0: # We don't have to compute anything, A is empty. # We need this special case because Numpy considers it diff --git a/pytensor/tensor/rewriting/blas_scipy.py b/pytensor/tensor/rewriting/blas_scipy.py index 610ef9b82f..2ed0279e45 100644 --- a/pytensor/tensor/rewriting/blas_scipy.py +++ b/pytensor/tensor/rewriting/blas_scipy.py @@ -1,5 +1,5 @@ from pytensor.graph.rewriting.basic import in2out -from pytensor.tensor.blas import ger, ger_destructive, have_fblas +from pytensor.tensor.blas import ger, ger_destructive from pytensor.tensor.blas_scipy import scipy_ger_inplace, scipy_ger_no_inplace from pytensor.tensor.rewriting.blas import blas_optdb, node_rewriter, optdb @@ -19,19 +19,19 @@ def make_ger_destructive(fgraph, node): use_scipy_blas = in2out(use_scipy_ger) make_scipy_blas_destructive = in2out(make_ger_destructive) -if have_fblas: - # scipy_blas is scheduled in the blas_optdb very late, because scipy sortof - # sucks, but it is almost always present. - # C implementations should be scheduled earlier than this, so that they take - # precedence. Once the original Ger is replaced, then these optimizations - # have no effect. - blas_optdb.register("scipy_blas", use_scipy_blas, "fast_run", position=100) - - # this matches the InplaceBlasOpt defined in blas.py - optdb.register( - "make_scipy_blas_destructive", - make_scipy_blas_destructive, - "fast_run", - "inplace", - position=50.2, - ) + +# scipy_blas is scheduled in the blas_optdb very late, because scipy sortof +# sucks [citation needed], but it is almost always present. +# C implementations should be scheduled earlier than this, so that they take +# precedence. Once the original Ger is replaced, then these optimizations +# have no effect. +blas_optdb.register("scipy_blas", use_scipy_blas, "fast_run", position=100) + +# this matches the InplaceBlasOpt defined in blas.py +optdb.register( + "make_scipy_blas_destructive", + make_scipy_blas_destructive, + "fast_run", + "inplace", + position=50.2, +) diff --git a/tests/d3viz/test_d3viz.py b/tests/d3viz/test_d3viz.py index b6b6479a1b..7e4b0426a0 100644 --- a/tests/d3viz/test_d3viz.py +++ b/tests/d3viz/test_d3viz.py @@ -9,12 +9,14 @@ from pytensor import compile from pytensor.compile.function import function from pytensor.configdefaults import config -from pytensor.printing import pydot_imported, pydot_imported_msg +from pytensor.printing import _try_pydot_import from tests.d3viz import models -if not pydot_imported: - pytest.skip("pydot not available: " + pydot_imported_msg, allow_module_level=True) +try: + _try_pydot_import() +except Exception as e: + pytest.skip(f"pydot not available: {e!s}", allow_module_level=True) class TestD3Viz: diff --git a/tests/d3viz/test_formatting.py b/tests/d3viz/test_formatting.py index 9f5f8be9ec..7d1149be0e 100644 --- a/tests/d3viz/test_formatting.py +++ b/tests/d3viz/test_formatting.py @@ -3,11 +3,13 @@ from pytensor import config, function from pytensor.d3viz.formatting import PyDotFormatter -from pytensor.printing import pydot_imported, pydot_imported_msg +from pytensor.printing import _try_pydot_import -if not pydot_imported: - pytest.skip("pydot not available: " + pydot_imported_msg, allow_module_level=True) +try: + _try_pydot_import() +except Exception as e: + pytest.skip(f"pydot not available: {e!s}", allow_module_level=True) from tests.d3viz import models diff --git a/tests/scan/test_printing.py b/tests/scan/test_printing.py index 44465f0152..9bf32af48f 100644 --- a/tests/scan/test_printing.py +++ b/tests/scan/test_printing.py @@ -5,7 +5,7 @@ import pytensor.tensor as pt from pytensor.configdefaults import config from pytensor.graph.fg import FunctionGraph -from pytensor.printing import debugprint, pydot_imported, pydotprint +from pytensor.printing import _try_pydot_import, debugprint, pydotprint from pytensor.tensor.type import dvector, iscalar, scalar, vector @@ -686,6 +686,13 @@ def no_shared_fn(n, x_tm1, M): assert truth.strip() == out.strip() +try: + _try_pydot_import() + pydot_imported = True +except Exception: + pydot_imported = False + + @pytest.mark.skipif(not pydot_imported, reason="pydot not available") def test_pydotprint(): def f_pow2(x_tm1): diff --git a/tests/tensor/test_blas_scipy.py b/tests/tensor/test_blas_scipy.py index 7cdfaadc34..716eab7bbe 100644 --- a/tests/tensor/test_blas_scipy.py +++ b/tests/tensor/test_blas_scipy.py @@ -1,7 +1,6 @@ import pickle import numpy as np -import pytest import pytensor from pytensor import tensor as pt @@ -12,7 +11,6 @@ from tests.unittest_tools import OptimizationTestMixin -@pytest.mark.skipif(not pytensor.tensor.blas_scipy.have_fblas, reason="fblas needed") class TestScipyGer(OptimizationTestMixin): def setup_method(self): self.mode = pytensor.compile.get_default_mode() diff --git a/tests/test_printing.py b/tests/test_printing.py index 73403880e9..be5dbbc5a1 100644 --- a/tests/test_printing.py +++ b/tests/test_printing.py @@ -17,13 +17,13 @@ PatternPrinter, PPrinter, Print, + _try_pydot_import, char_from_number, debugprint, default_printer, get_node_by_id, min_informative_str, pp, - pydot_imported, pydotprint, ) from pytensor.tensor import as_tensor_variable @@ -31,6 +31,13 @@ from tests.graph.utils import MyInnerGraphOp, MyOp, MyVariable +try: + _try_pydot_import() + pydot_imported = True +except Exception: + pydot_imported = False + + @pytest.mark.parametrize( "number,s", [