From 5290c8e0a8f5b0d0e7053fe0eeeb7181153e71e7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 21 Aug 2018 23:21:19 -0700 Subject: [PATCH 01/19] Avoid the cryptic error about fake-julia/../lib/julia/sys.so --- julia/core.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/julia/core.py b/julia/core.py index 95a889e4..c5bb1f45 100644 --- a/julia/core.py +++ b/julia/core.py @@ -42,6 +42,12 @@ def iteritems(d): return iter(d.items()) else: iteritems = dict.iteritems + +# As setting up Julia modifies os.environ, we need to cache it for +# launching subprocesses later in the original environment. +_enviorn = os.environ.copy() + + class JuliaError(Exception): pass @@ -278,7 +284,11 @@ def juliainfo(runtime='julia'): include(PyCall_depsfile) println(pyprogramname) end - """]) + """], + # Use the original environment variables to avoid a cryptic + # error "fake-julia/../lib/julia/sys.so: cannot open shared + # object file: No such file or directory": + env=_enviorn) args = output.decode("utf-8").rstrip().split("\n") if len(args) == 3: args.append(None) # no pyprogramname set From d49d819d7a5343998e7314010635ede278b2190f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 21 Aug 2018 23:52:07 -0700 Subject: [PATCH 02/19] More robust pre-compilation cache handling for Julia 0.6 fixes #175 --- julia/core.py | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/julia/core.py b/julia/core.py index c5bb1f45..27c82779 100644 --- a/julia/core.py +++ b/julia/core.py @@ -451,16 +451,37 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, # configuration and so do any packages that depend on it. self._call(u"unshift!(Base.LOAD_CACHE_PATH, abspath(Pkg.Dir._pkgroot()," + "\"lib\", \"pyjulia%s-v$(VERSION.major).$(VERSION.minor)\"))" % sys.version_info[0]) - # If PyCall.ji does not exist, create an empty file to force - # recompilation + + # If PyCall.jl is already pre-compiled, for the global + # environment, hide it while we are loading PyCall.jl + # for PyJulia which has to compile a new cache if it + # does not exist. However, Julia does not compile a + # new cache if it exists in Base.LOAD_CACHE_PATH[2:end]. + # https://github.com/JuliaPy/pyjulia/issues/92#issuecomment-289303684 self._call(u""" - isdir(Base.LOAD_CACHE_PATH[1]) || - mkpath(Base.LOAD_CACHE_PATH[1]) - depsfile = joinpath(Base.LOAD_CACHE_PATH[1],"PyCall.ji") - isfile(depsfile) || touch(depsfile) + for path in Base.LOAD_CACHE_PATH[2:end] + cache = joinpath(path, "PyCall.ji") + backup = joinpath(path, "PyCall.ji.backup") + if isfile(cache) + mv(cache, backup; remove_destination=true) + end + end """) self._call(u"using PyCall") + + if use_separate_cache: + self._call(u""" + for path in Base.LOAD_CACHE_PATH[2:end] + cache = joinpath(path, "PyCall.ji") + backup = joinpath(path, "PyCall.ji.backup") + if !isfile(cache) && isfile(backup) + mv(backup, cache) + end + rm(backup; force=true) + end + """) + # Whether we initialized Julia or not, we MUST create at least one # instance of PyObject and the convert function. Since these will be # needed on every call, we hold them in the Julia object itself so From a748dea306f52432ff9927a05f3365e132a2f41e Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 21 Aug 2018 22:07:39 -0700 Subject: [PATCH 03/19] Run doctest via pytest --- tox.ini | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tox.ini b/tox.ini index 459153cc..90d2e21a 100644 --- a/tox.ini +++ b/tox.ini @@ -27,3 +27,9 @@ passenv = # See: https://coveralls-python.readthedocs.io/en/latest/usage/tox.html#travisci TRAVIS TRAVIS_* + +[pytest] +addopts = + --doctest-modules + --ignore=julia/fake-julia + --ignore=test/_star_import.py From 54d1116c4bbf3bd7a223450eb9093e4d53d2072c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 17:35:35 -0700 Subject: [PATCH 04/19] Don't install pytest 3.7.3 to avoid doctest failures in macOS --- tox.ini | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 90d2e21a..05ea3c67 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,10 @@ envlist = py27, py36 [testenv] deps = - pytest + pytest != 3.7.3 + # Don't install 3.7.3 to avoid doctest failures in macOS + # https://github.com/pytest-dev/pytest/pull/3893#issuecomment-416560866 + pytest-cov numpy ipython From 1825d3443afce018e67d2c2e7390b6bb2d26a82b Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 18:35:38 -0700 Subject: [PATCH 05/19] Make PyJulia usable in virtual environments --- julia/core.py | 47 +++++++--- julia/find_libpython.py | 199 ++++++++++++++++++++++++++++++++++++++++ test/test_utils.py | 16 +--- 3 files changed, 235 insertions(+), 27 deletions(-) create mode 100755 julia/find_libpython.py diff --git a/julia/core.py b/julia/core.py index 27c82779..50f03b53 100644 --- a/julia/core.py +++ b/julia/core.py @@ -32,6 +32,8 @@ # this is python 3.3 specific from types import ModuleType, FunctionType +from .find_libpython import find_libpython, normalize_path + #----------------------------------------------------------------------------- # Classes and funtions #----------------------------------------------------------------------------- @@ -260,7 +262,11 @@ def determine_if_statically_linked(): JuliaInfo = namedtuple( 'JuliaInfo', - ['JULIA_HOME', 'libjulia_path', 'image_file', 'pyprogramname']) + ['JULIA_HOME', 'libjulia_path', 'image_file', + # Variables in PyCall/deps/deps.jl: + 'pyprogramname', 'libpython'], + # PyCall/deps/deps.jl may not exist; The variables are then set to None: + defaults=[None, None]) def juliainfo(runtime='julia'): @@ -283,6 +289,7 @@ def juliainfo(runtime='julia'): if PyCall_depsfile !== nothing && isfile(PyCall_depsfile) include(PyCall_depsfile) println(pyprogramname) + println(libpython) end """], # Use the original environment variables to avoid a cryptic @@ -290,21 +297,31 @@ def juliainfo(runtime='julia'): # object file: No such file or directory": env=_enviorn) args = output.decode("utf-8").rstrip().split("\n") - if len(args) == 3: - args.append(None) # no pyprogramname set return JuliaInfo(*args) -def is_same_path(a, b): - a = os.path.normpath(os.path.normcase(a)) - b = os.path.normpath(os.path.normcase(b)) - return a == b +def is_compatible_exe(jlinfo): + """ + Determine if Python used by PyCall.jl is compatible with this Python. + + Current Python executable is considered compatible if it is dynamically + linked to libpython (usually the case in macOS and Windows) and + both of them are using identical libpython. If this function returns + `True`, PyJulia use the same precompilation cache of PyCall.jl used by + Julia itself. + + Parameters + ---------- + jlinfo : JuliaInfo + A `JuliaInfo` object returned by `juliainfo` function. + """ + if jlinfo.libpython is None: + return False + if determine_if_statically_linked(): + return False -def is_different_exe(pyprogramname, sys_executable): - if pyprogramname is None: - return True - return not is_same_path(pyprogramname, sys_executable) + return find_libpython() == normalize_path(jlinfo.libpython) _julia_runtime = [False] @@ -359,11 +376,10 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, runtime = jl_runtime_path else: runtime = 'julia' - JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo(runtime) + jlinfo = juliainfo(runtime) + JULIA_HOME, libjulia_path, image_file, depsjlexe = jlinfo[:4] self._debug("pyprogramname =", depsjlexe) self._debug("sys.executable =", sys.executable) - exe_differs = is_different_exe(depsjlexe, sys.executable) - self._debug("exe_differs =", exe_differs) self._debug("JULIA_HOME = %s, libjulia_path = %s" % (JULIA_HOME, libjulia_path)) if not os.path.exists(libjulia_path): raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path)) @@ -381,7 +397,8 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, else: jl_init_path = JULIA_HOME.encode("utf-8") # initialize with JULIA_HOME - use_separate_cache = exe_differs or determine_if_statically_linked() + use_separate_cache = not is_compatible_exe(jlinfo) + self._debug("use_separate_cache =", use_separate_cache) if use_separate_cache: PYCALL_JULIA_HOME = os.path.join( os.path.dirname(os.path.realpath(__file__)),"fake-julia").replace("\\","\\\\") diff --git a/julia/find_libpython.py b/julia/find_libpython.py new file mode 100755 index 00000000..686aa736 --- /dev/null +++ b/julia/find_libpython.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python + +""" +Locate libpython associated with this Python executable. +""" + +from __future__ import print_function + +from logging import getLogger +import ctypes.util +import os +import platform +import sys +import sysconfig + +logger = getLogger("find_libpython") + +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so" + + +def library_name(name, suffix=SHLIB_SUFFIX, + is_windows=platform.system() == "Windows"): + """ + Convert a file basename `name` to a library name (no "lib" and ".so" etc.) + + >>> library_name("libpython3.7m.so") # doctest: +SKIP + 'python3.7m' + >>> library_name("libpython3.7m.so", suffix=".so", is_windows=False) + 'python3.7m' + >>> library_name("libpython3.7m.dylib", suffix=".dylib", is_windows=False) + 'python3.7m' + >>> library_name("python37.dll", suffix=".dll", is_windows=True) + 'python37' + """ + if not is_windows: + name = name[len("lib"):] + if suffix and name.endswith(suffix): + name = name[:-len(suffix)] + return name + + +def append_truthy(list, item): + if item: + list.append(item) + + +def libpython_candidates(suffix=SHLIB_SUFFIX): + """ + Iterate over candidate paths of libpython. + + Yields + ------ + path : str or None + Candidate path to libpython. The path may not be a fullpath + and may not exist. + """ + is_windows = platform.system() == "Windows" + + # List candidates for libpython basenames + lib_basenames = [] + append_truthy(lib_basenames, sysconfig.get_config_var("LDLIBRARY")) + + LIBRARY = sysconfig.get_config_var("LIBRARY") + if LIBRARY: + lib_basenames.append(os.path.splitext(LIBRARY)[0] + suffix) + + dlprefix = "" if is_windows else "lib" + sysdata = dict( + v=sys.version_info, + abiflags=(sysconfig.get_config_var("ABIFLAGS") or + sysconfig.get_config_var("abiflags") or ""), + ) + lib_basenames.extend(dlprefix + p + suffix for p in [ + "python{v.major}.{v.minor}{abiflags}".format(**sysdata), + "python{v.major}.{v.minor}".format(**sysdata), + "python{v.major}".format(**sysdata), + "python", + ]) + + # List candidates for directories in which libpython may exist + lib_dirs = [] + append_truthy(lib_dirs, sysconfig.get_config_var("LIBDIR")) + + if is_windows: + lib_dirs.append(os.path.join(os.path.dirname(sys.executable))) + else: + lib_dirs.append(os.path.join( + os.path.dirname(os.path.dirname(sys.executable)), + "lib")) + + # For macOS: + append_truthy(lib_dirs, sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX")) + + lib_dirs.append(sys.exec_prefix) + lib_dirs.append(os.path.join(sys.exec_prefix, "lib")) + + for directory in lib_dirs: + for basename in lib_basenames: + yield os.path.join(directory, basename) + + # In macOS and Windows, ctypes.util.find_library returns a full path: + for basename in lib_basenames: + yield ctypes.util.find_library(library_name(basename)) + + +def normalize_path(path, suffix=SHLIB_SUFFIX): + """ + Normalize shared library `path` to a real path. + + If `path` is not a full path, `None` is returned. If `path` does + not exists, append `SHLIB_SUFFIX` and check if it exists. + Finally, the path is canonicalized by following the symlinks. + + Parameters + ---------- + path : str ot None + A candidate path to a shared library. + """ + if not path: + return None + if not os.path.isabs(path): + return None + if os.path.exists(path): + return os.path.realpath(path) + if os.path.exists(path + suffix): + return os.path.realpath(path + suffix) + return None + + +def finding_libpython(): + """ + Iterate over existing libpython paths. + + The first item is likely to be the best one. It may yield + duplicated paths. + + Yields + ------ + path : str + Existing path to a libpython. + """ + for path in libpython_candidates(): + logger.debug("Candidate: %s", path) + normalized = normalize_path(path) + logger.debug("Normalized: %s", normalized) + if normalized: + logger.debug("Found: %s", normalized) + yield normalized + + +def find_libpython(): + """ + Return a path (`str`) to libpython or `None` if not found. + + Parameters + ---------- + path : str or None + Existing path to the (supposedly) correct libpython. + """ + for path in finding_libpython(): + return os.path.realpath(path) + + +def cli_find_libpython(verbose, list_all): + import logging + # Importing `logging` module here so that using `logging.debug` + # instead of `logger.debug` outside of this function becomes an + # error. + + if verbose: + logging.basicConfig(level=logging.DEBUG) + + if list_all: + for path in finding_libpython(): + print(path) + return + + path = find_libpython() + if path is None: + return 1 + print(path, end="") + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + description=__doc__) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Print debugging information.") + parser.add_argument( + "--list-all", action="store_true", + help="Print list of all paths found.") + ns = parser.parse_args(args) + parser.exit(cli_find_libpython(**vars(ns))) + + +if __name__ == "__main__": + main() diff --git a/test/test_utils.py b/test/test_utils.py index a86d146b..a8c15c88 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -2,17 +2,9 @@ Unit tests which can be done without loading `libjulia`. """ -import sys +from julia.find_libpython import finding_libpython -import pytest -from julia.core import is_different_exe - - -@pytest.mark.parametrize('pyprogramname, sys_executable, exe_differs', [ - (sys.executable, sys.executable, False), - (None, sys.executable, True), - ('/dev/null', sys.executable, True), -]) -def test_is_different_exe(pyprogramname, sys_executable, exe_differs): - assert is_different_exe(pyprogramname, sys_executable) == exe_differs +def test_smoke_finding_libpython(): + paths = list(finding_libpython()) + assert set(map(type, paths)) == {str} From 1da62436f7a898c3684084b9ee64cae2564a91b9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 22:14:26 -0700 Subject: [PATCH 06/19] Python < 3.7 does not support namedtuple(..., defaults=...) --- julia/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/julia/core.py b/julia/core.py index 50f03b53..7ea7cdff 100644 --- a/julia/core.py +++ b/julia/core.py @@ -264,9 +264,7 @@ def determine_if_statically_linked(): 'JuliaInfo', ['JULIA_HOME', 'libjulia_path', 'image_file', # Variables in PyCall/deps/deps.jl: - 'pyprogramname', 'libpython'], - # PyCall/deps/deps.jl may not exist; The variables are then set to None: - defaults=[None, None]) + 'pyprogramname', 'libpython']) def juliainfo(runtime='julia'): @@ -297,6 +295,7 @@ def juliainfo(runtime='julia'): # object file: No such file or directory": env=_enviorn) args = output.decode("utf-8").rstrip().split("\n") + args.extend([None] * (len(JuliaInfo._fields) - len(args))) return JuliaInfo(*args) From 252cd5523ecc95bc9d7947085bd34154eba36936 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 22:40:54 -0700 Subject: [PATCH 07/19] Print debug message from is_compatible_exe --- julia/core.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/julia/core.py b/julia/core.py index 7ea7cdff..9b793abf 100644 --- a/julia/core.py +++ b/julia/core.py @@ -299,7 +299,7 @@ def juliainfo(runtime='julia'): return JuliaInfo(*args) -def is_compatible_exe(jlinfo): +def is_compatible_exe(jlinfo, _debug=lambda *_: None): """ Determine if Python used by PyCall.jl is compatible with this Python. @@ -315,12 +315,18 @@ def is_compatible_exe(jlinfo): A `JuliaInfo` object returned by `juliainfo` function. """ if jlinfo.libpython is None: + _debug("libpython cannot be read from PyCall/deps/deps.jl") return False if determine_if_statically_linked(): + _debug(sys.executable, "is statically linked.") return False - return find_libpython() == normalize_path(jlinfo.libpython) + py_libpython = find_libpython() + jl_libpython = normalize_path(jlinfo.libpython) + _debug("py_libpython =", py_libpython) + _debug("jl_libpython =", jl_libpython) + return py_libpython == jl_libpython _julia_runtime = [False] @@ -396,7 +402,7 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, else: jl_init_path = JULIA_HOME.encode("utf-8") # initialize with JULIA_HOME - use_separate_cache = not is_compatible_exe(jlinfo) + use_separate_cache = not is_compatible_exe(jlinfo, _debug=self._debug) self._debug("use_separate_cache =", use_separate_cache) if use_separate_cache: PYCALL_JULIA_HOME = os.path.join( From 2ab64e77d3ab77caa816564fbd57d5875001dab7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 22:50:12 -0700 Subject: [PATCH 08/19] More permissive test for finding_libpython --- test/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index a8c15c88..c08601a0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -5,6 +5,8 @@ from julia.find_libpython import finding_libpython -def test_smoke_finding_libpython(): +def test_finding_libpython_yield_type(): paths = list(finding_libpython()) - assert set(map(type, paths)) == {str} + assert set(map(type, paths)) <= {str} +# In a statically linked Python executable, no paths may be found. So +# let's just check returned type of finding_libpython. From 1f4740439cd60d4a4323cdf93ac202f77dbdc996 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 22:52:57 -0700 Subject: [PATCH 09/19] Run find_libpython in tox --- tox.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tox.ini b/tox.ini index 05ea3c67..2713953e 100644 --- a/tox.ini +++ b/tox.ini @@ -12,6 +12,10 @@ deps = ipython mock commands = + python -m julia.find_libpython --list-all --verbose + # Print libpython candidates found by `find_libpython`. It may be + # useful for debugging. + python -m julia.with_rebuilt -- python -m pytest {posargs} # Using "python -m pytest" to exactly match the Python interpreter # used to build PyCall.jl via julia/with_rebuilt.py (when From 9adb0740817daf88e3412440957933299a8f0568 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 22:59:58 -0700 Subject: [PATCH 10/19] Use absolute_import in find_libpython.py --- julia/find_libpython.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 686aa736..92402c7d 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -4,7 +4,7 @@ Locate libpython associated with this Python executable. """ -from __future__ import print_function +from __future__ import print_function, absolute_import from logging import getLogger import ctypes.util From 6e91cf9371545643c2edd52b5559387617eec1cd Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 23:20:00 -0700 Subject: [PATCH 11/19] Print jlinfo.libpython --- julia/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/julia/core.py b/julia/core.py index 9b793abf..d2f0c54c 100644 --- a/julia/core.py +++ b/julia/core.py @@ -314,6 +314,7 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): jlinfo : JuliaInfo A `JuliaInfo` object returned by `juliainfo` function. """ + _debug("jlinfo.libpython =", jlinfo.libpython) if jlinfo.libpython is None: _debug("libpython cannot be read from PyCall/deps/deps.jl") return False From 435ccf4f7e22dcf2f944921d5bc94cc188e60ee7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 28 Aug 2018 23:41:55 -0700 Subject: [PATCH 12/19] Find the linked libpython using dladdr --- julia/find_libpython.py | 45 +++++++++++++++++++++++++++++++++++++++++ test/test_utils.py | 21 +++++++++++++++++-- 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 92402c7d..2281e456 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -18,6 +18,48 @@ SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so" +def linked_libpython(): + """ + Find the linked libpython using dladdr (in *nix). + + Calling this in Windows always return `None` at the moment. + + Returns + ------- + path : str or None + A path to linked libpython. Return `None` if statically linked. + """ + if platform.system() == "Windows": + return None + return _linked_libpython_unix() + + +class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + + +def _linked_libpython_unix(): + libdl = ctypes.CDLL(ctypes.util.find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + + dlinfo = Dl_info() + retcode = libdl.dladdr( + ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), + ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + path = os.path.realpath(dlinfo.dli_fname.decode()) + if path == os.path.realpath(sys.executable): + return None + return path + + def library_name(name, suffix=SHLIB_SUFFIX, is_windows=platform.system() == "Windows"): """ @@ -54,6 +96,9 @@ def libpython_candidates(suffix=SHLIB_SUFFIX): Candidate path to libpython. The path may not be a fullpath and may not exist. """ + + yield linked_libpython() + is_windows = platform.system() == "Windows" # List candidates for libpython basenames diff --git a/test/test_utils.py b/test/test_utils.py index c08601a0..0b6dc5c3 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -2,11 +2,28 @@ Unit tests which can be done without loading `libjulia`. """ -from julia.find_libpython import finding_libpython +import platform + +import pytest + +from julia.find_libpython import finding_libpython, linked_libpython +from julia.core import determine_if_statically_linked + +try: + unicode +except NameError: + unicode = str # for Python 3 def test_finding_libpython_yield_type(): paths = list(finding_libpython()) - assert set(map(type, paths)) <= {str} + assert set(map(type, paths)) <= {str, unicode} # In a statically linked Python executable, no paths may be found. So # let's just check returned type of finding_libpython. + + +@pytest.mark.xfail(platform.system() == "Windows", + reason="linked_libpython is not implemented for Windows") +def test_linked_libpython(): + if determine_if_statically_linked(): + assert linked_libpython() is not None From 25631af6061e548c16cef6cc2e5aa736b66d2844 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 00:45:57 -0700 Subject: [PATCH 13/19] Manually set SHLIB_SUFFIX in non-Linux platforms --- julia/find_libpython.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 2281e456..1c53ef36 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -15,7 +15,14 @@ logger = getLogger("find_libpython") -SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") or ".so" +SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") +if SHLIB_SUFFIX is None: + if platform.system() == "Windows": + SHLIB_SUFFIX = ".dll" + elif platform.system() == "Darwin": + SHLIB_SUFFIX = ".dylib" + else: + SHLIB_SUFFIX = ".so" def linked_libpython(): From e98c265c44ce05d9903d8aee563335e493acde3b Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 00:50:07 -0700 Subject: [PATCH 14/19] Use sysconfig.get_config_var("VERSION") --- julia/find_libpython.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 1c53ef36..285a44e1 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -119,12 +119,15 @@ def libpython_candidates(suffix=SHLIB_SUFFIX): dlprefix = "" if is_windows else "lib" sysdata = dict( v=sys.version_info, - abiflags=(sysconfig.get_config_var("ABIFLAGS") or + # VERSION is X.Y in Linux/macOS and XY in Windows: + VERSION=(sysconfig.get_config_var("VERSION") or + "{v.major}.{v.minor}".format(v=sys.version_info)), + ABIFLAGS=(sysconfig.get_config_var("ABIFLAGS") or sysconfig.get_config_var("abiflags") or ""), ) lib_basenames.extend(dlprefix + p + suffix for p in [ - "python{v.major}.{v.minor}{abiflags}".format(**sysdata), - "python{v.major}.{v.minor}".format(**sysdata), + "python{VERSION}{ABIFLAGS}".format(**sysdata), + "python{VERSION}".format(**sysdata), "python{v.major}".format(**sysdata), "python", ]) From f09ae4256b1901a2d31e334263e5e4ef9d1c026f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 01:12:04 -0700 Subject: [PATCH 15/19] Directly compare executable paths --- julia/core.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/julia/core.py b/julia/core.py index d2f0c54c..48d1d32d 100644 --- a/julia/core.py +++ b/julia/core.py @@ -299,6 +299,12 @@ def juliainfo(runtime='julia'): return JuliaInfo(*args) +def is_same_path(a, b): + a = os.path.realpath(os.path.normcase(a)) + b = os.path.realpath(os.path.normcase(b)) + return a == b + + def is_compatible_exe(jlinfo, _debug=lambda *_: None): """ Determine if Python used by PyCall.jl is compatible with this Python. @@ -323,6 +329,14 @@ def is_compatible_exe(jlinfo, _debug=lambda *_: None): _debug(sys.executable, "is statically linked.") return False + # Note that the following check is OK since statically linked case + # is already excluded. + if is_same_path(jlinfo.pyprogramname, sys.executable): + # In macOS and Windows, find_libpython does not work as good + # as in Linux. We add this shortcut so that PyJulia can work + # in those environments. + return True + py_libpython = find_libpython() jl_libpython = normalize_path(jlinfo.libpython) _debug("py_libpython =", py_libpython) From 96429c64f3de75fbb437b39b03d6482d54c8d9aa Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 02:03:54 -0700 Subject: [PATCH 16/19] Print PyCall/deps/build.log --- julia/with_rebuilt.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/julia/with_rebuilt.py b/julia/with_rebuilt.py index 48ce7122..f483b930 100644 --- a/julia/with_rebuilt.py +++ b/julia/with_rebuilt.py @@ -27,6 +27,16 @@ def maybe_rebuild(rebuild, julia): using Pkg end Pkg.build("PyCall") + if VERSION < v"0.7.0" + pkgdir = Pkg.dir("PyCall") + else + modpath = Base.locate_package(Base.identify_package("PyCall")) + pkgdir = joinpath(dirname(modpath), "..") + end + logfile = joinpath(pkgdir, "deps", "build.log") + if isfile(logfile) + print(read(logfile, String)) + end """] print('Building PyCall.jl with PYTHON =', sys.executable) print(*build) From 6982ae885989285fc58c4b5181071b79975ea7f7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 02:10:34 -0700 Subject: [PATCH 17/19] Test against 1.0 instead of nightly in AppVeyor --- appveyor.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 85651094..11d057f5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -13,22 +13,32 @@ environment: BATDIR: ci\appveyor\win32 CROSS_VERSION: 1 - # 32 julia latest Python-35 - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + # 32 julia-1.0 Python-35 + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/1.0/julia-1.0-latest-win32.exe" PYTHONDIR: "C:\\Python35" BATDIR: ci\appveyor\win32 + # 32 julia latest Python-35 + # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + # PYTHONDIR: "C:\\Python35" + # BATDIR: ci\appveyor\win32 + # 64 julia-0.6 Python-35 - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" PYTHONDIR: "C:\\Python35-x64" BATDIR: ci\appveyor\win64 CROSS_VERSION: 1 - # 64 julia latest Python-35 - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" + # 64 julia-1.0 Python-35 + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0-latest-win64.exe" PYTHONDIR: "C:\\Python35-x64" BATDIR: ci\appveyor\win64 + # 64 julia latest Python-35 + # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" + # PYTHONDIR: "C:\\Python35-x64" + # BATDIR: ci\appveyor\win64 + matrix: allow_failures: - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" From fe0cd97b27c950534c0c8f84223b3ae5d91a219c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 15:25:25 -0700 Subject: [PATCH 18/19] Do not use platform.system() os.name and sys.platform are more well-defined/documented and we don't need more detail than them. --- julia/find_libpython.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 285a44e1..18bbc3f2 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -9,17 +9,19 @@ from logging import getLogger import ctypes.util import os -import platform import sys import sysconfig logger = getLogger("find_libpython") +is_windows = os.name == "nt" +is_apple = sys.platform == "darwin" + SHLIB_SUFFIX = sysconfig.get_config_var("SHLIB_SUFFIX") if SHLIB_SUFFIX is None: - if platform.system() == "Windows": + if is_windows: SHLIB_SUFFIX = ".dll" - elif platform.system() == "Darwin": + elif is_apple: SHLIB_SUFFIX = ".dylib" else: SHLIB_SUFFIX = ".so" @@ -36,7 +38,7 @@ def linked_libpython(): path : str or None A path to linked libpython. Return `None` if statically linked. """ - if platform.system() == "Windows": + if is_windows: return None return _linked_libpython_unix() @@ -67,8 +69,7 @@ def _linked_libpython_unix(): return path -def library_name(name, suffix=SHLIB_SUFFIX, - is_windows=platform.system() == "Windows"): +def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): """ Convert a file basename `name` to a library name (no "lib" and ".so" etc.) @@ -106,8 +107,6 @@ def libpython_candidates(suffix=SHLIB_SUFFIX): yield linked_libpython() - is_windows = platform.system() == "Windows" - # List candidates for libpython basenames lib_basenames = [] append_truthy(lib_basenames, sysconfig.get_config_var("LDLIBRARY")) @@ -194,6 +193,8 @@ def finding_libpython(): path : str Existing path to a libpython. """ + logger.debug("is_windows = %s", is_windows) + logger.debug("is_apple = %s", is_apple) for path in libpython_candidates(): logger.debug("Candidate: %s", path) normalized = normalize_path(path) From cdbefb19453e29c6b64f9c4b96857c4d441bb171 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 29 Aug 2018 15:48:34 -0700 Subject: [PATCH 19/19] More manual suffix handling for macOS --- julia/find_libpython.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/julia/find_libpython.py b/julia/find_libpython.py index 18bbc3f2..7fec1099 100755 --- a/julia/find_libpython.py +++ b/julia/find_libpython.py @@ -21,10 +21,12 @@ if SHLIB_SUFFIX is None: if is_windows: SHLIB_SUFFIX = ".dll" - elif is_apple: - SHLIB_SUFFIX = ".dylib" else: SHLIB_SUFFIX = ".so" +if is_apple: + # sysconfig.get_config_var("SHLIB_SUFFIX") can be ".so" in macOS. + # Let's not use the value from sysconfig. + SHLIB_SUFFIX = ".dylib" def linked_libpython(): @@ -157,7 +159,7 @@ def libpython_candidates(suffix=SHLIB_SUFFIX): yield ctypes.util.find_library(library_name(basename)) -def normalize_path(path, suffix=SHLIB_SUFFIX): +def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): """ Normalize shared library `path` to a real path. @@ -178,9 +180,31 @@ def normalize_path(path, suffix=SHLIB_SUFFIX): return os.path.realpath(path) if os.path.exists(path + suffix): return os.path.realpath(path + suffix) + if is_apple: + return normalize_path(_remove_suffix_apple(path), + suffix=".so", is_apple=False) return None +def _remove_suffix_apple(path): + """ + Strip off .so or .dylib. + + >>> _remove_suffix_apple("libpython.so") + 'libpython' + >>> _remove_suffix_apple("libpython.dylib") + 'libpython' + >>> _remove_suffix_apple("libpython3.7") + 'libpython3.7' + """ + if path.endswith(".dylib"): + return path[:-len(".dylib")] + if path.endswith(".so"): + return path[:-len(".so")] + return path + + + def finding_libpython(): """ Iterate over existing libpython paths.