diff --git a/.gitignore b/.gitignore index 03f13b0d..37dad702 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ pip-log.txt # Unit test / coverage reports .coverage +.pytest_cache .tox nosetests.xml diff --git a/.travis.yml b/.travis.yml index bd106ecd..127b5324 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,54 +1,43 @@ +language: python +os: + - linux +python: + - "2.7" + - "3.6" +env: + matrix: + - JULIA_VERSION=0.6.4 CROSS_VERSION=1 + - JULIA_VERSION=0.7.0-rc2 + # - JULIA_VERSION=nightly + global: + - TOXENV=py matrix: # Python environment is not functional on OS X include: - - language: python - python: 2.7 - env: JULIA_VERSION=juliareleases - os: linux - - language: python - python: 2.7 - env: JULIA_VERSION=julianightlies - os: linux - - language: python - python: 3.5 - env: - - JULIA_VERSION=juliareleases - - CROSS_VERSION=1 - os: linux - - language: python - python: 3.5 - env: JULIA_VERSION=julianightlies - os: linux - - language: python - python: 3.5 - env: - - JULIA_VERSION=julianightlies - - CROSS_VERSION=1 - os: linux - language: generic env: - PYTHON=python2 - - JULIA_VERSION=julianightlies + - JULIA_VERSION=0.7.0-rc2 + # - JULIA_VERSION=nightly os: osx - language: generic env: - PYTHON=python2 - - JULIA_VERSION=juliareleases + - JULIA_VERSION=0.6.4 + - CROSS_VERSION=1 os: osx - language: generic env: - PYTHON=python3 - - JULIA_VERSION=julianightlies + - JULIA_VERSION=0.7.0-rc2 + # - JULIA_VERSION=nightly os: osx - language: generic env: - PYTHON=python3 - - JULIA_VERSION=juliareleases - os: osx - allow_failures: - - env: - - JULIA_VERSION=julianightlies + - JULIA_VERSION=0.6.4 - CROSS_VERSION=1 + os: osx branches: only: - master @@ -57,19 +46,21 @@ notifications: before_script: - echo ./ci/install-julia.sh "$JULIA_VERSION" - ./ci/install-julia.sh "$JULIA_VERSION" - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq -y; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install libpcre3-dev -y; fi - - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get install python-numpy python3-numpy -y; fi - if [ "$TRAVIS_OS_NAME" = "osx" -a "$PYTHON" = "python3" ]; then brew update; brew upgrade python || echo "Ignoring errors..."; fi - if [ "$TRAVIS_OS_NAME" = "osx" -a "$PYTHON" = "python2" ]; then brew update; brew list python@2 &>/dev/null || brew install python@2 || echo "Ignoring errors..."; fi # Ignoring errors from brew since it may actually be OK to do so. # Following which command will catch installation failure: - - which ${PYTHON:-python} -script: - - julia -e 'Pkg.add("PyCall")' - - /usr/bin/python --version - PYTHON=${PYTHON:-python} - - echo $PYTHON - - $PYTHON --version - - if [ "$CROSS_VERSION" = "1" ]; then /usr/bin/python -m unittest discover; fi - - $PYTHON -m unittest discover + - which $PYTHON + - $PYTHON -m pip --version + - $PYTHON -m pip install --quiet tox + - julia -e 'Pkg.add("PyCall")' +script: + + # "py,py27" below would be redundant when the main interpreter is + # Python 2.7 but it simplifies the CI setup. + - if [ "$CROSS_VERSION" = "1" ]; then + $PYTHON -m tox -e py,py27 -- -s; + fi + + - PYJULIA_TEST_REBUILD=yes $PYTHON -m tox -- -s diff --git a/README.md b/README.md index 83e99397..80b0de9d 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,14 @@ Experimenting with developing a better interface to [Julia language](https://jul to run the tests, execute from the toplevel directory ```shell -python -m unittest discover +tox ``` +See [Testing](#testing) below for details. + **Note** You need to explicitly add julia to your `PATH`, an alias will not work. -`pyjulia` is tested against Python versions 2.7 and 3.5. Older versions of Python (than 2.7) are not supported. +`pyjulia` is tested against Python versions 2.7, 3.6, and 3.7. Older versions of Python (than 2.7) are not supported. Installation ------------ @@ -138,3 +140,38 @@ Limitations ------------ Not all valid Julia identifiers are valid Python identifiers. Unicode identifiers are invalid in Python 2.7 and so `pyjulia` cannot call or access Julia methods/variables with names that are not ASCII only. Additionally, it is a common idiom in Julia to append a `!` character to methods which mutate their arguments. These method names are invalid Python identifers. `pyjulia` renames these methods by subsituting `!` with `_b`. For example, the Julia method `sum!` can be called in `pyjulia` using `sum_b(...)`. + + +Testing +------- + +The full syntax for invoking `tox` is + +```shell +[PYJULIA_TEST_REBUILD=yes] [JULIA_EXE=] tox [options] [-- pytest options] +``` + +* `PYJULIA_TEST_REBUILD`: *Be careful using this environment + variable!* When it is set to `yes`, your `PyCall.jl` installation + will be rebuilt using the Python interpreter used for testing. The + test suite tries to build back to the original configuration but the + precompilation would be in the stale state after the test. Note + also that it does not work if you unconditionally set `PYTHON` + environment variable in your Julia startup file. + +* `JULIA_EXE`: `julia` executable to be used for testing. + +* Positional arguments after `--` are passed to `pytest`. + +For example, + +```shell +PYJULIA_TEST_REBUILD=yes JULIA_EXE=~/julia/julia tox -e py37 -- -s +``` + +means to execute tests with + +* `pyjulia` in shared-cache mode +* `julia` executable at `~/julia/julia` +* Python 3.7 +* `pytest`'s capturing mode turned off diff --git a/appveyor.yml b/appveyor.yml index 9a45dbf1..193efc50 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,45 +1,38 @@ environment: + + TOXENV: py,py27 + TOX_TESTENV_PASSENV: DISTUTILS_USE_SDK MSSdk INCLUDE LIB + # https://packaging.python.org/guides/supporting-windows-using-appveyor/#testing-with-tox + # for more python versions have a look at # https://github.com/ogrisel/python-appveyor-demo/blob/master/appveyor.yml matrix: - # 64 julia-0.6 Python-27 - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - PYTHONDIR: "C:\\Python27-x64" - - # 32 julia-0.6 Python-27 + # 32 julia-0.6 Python-35 - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - PYTHONDIR: "C:\\Python27" + PYTHONDIR: "C:\\Python35" + BATDIR: ci\appveyor\win32 + CROSS_VERSION: 1 + + # 32 julia latest Python-35 + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + PYTHONDIR: "C:\\Python35" + BATDIR: ci\appveyor\win32 # 64 julia-0.6 Python-35 - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" PYTHONDIR: "C:\\Python35-x64" - - # 32 julia-latest Python-27 - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - PYTHONDIR: "C:\\Python27" - - # 64 julia-latest Python-27 - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" - PYTHONDIR: "C:\\Python27-x64" + BATDIR: ci\appveyor\win64 + CROSS_VERSION: 1 # 64 julia latest Python-35 - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" PYTHONDIR: "C:\\Python35-x64" - - # 64 julia latest Cross Version - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" - PYTHONDIR: "C:\\Python35-x64" - CROSS_VERSION_PATH: "C:\\Python27-x64" - - # 32 julia latest Cross Version - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - PYTHONDIR: "C:\\Python35" - CROSS_VERSION_PATH: "C:\\Python27" + BATDIR: ci\appveyor\win64 matrix: - allow_failures: - - CROSS_VERSION_PATH: "C:\\Python27-x64" - - CROSS_VERSION_PATH: "C:\\Python27" + allow_failures: + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" + - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" branches: only: @@ -71,11 +64,18 @@ build_script: # - C:\projects\julia\bin\julia -e "using PyCall; @assert isdefined(:PyCall); @assert typeof(PyCall) === Module" - "SET PYTHON=%PYTHONDIR%\\python.exe" - C:\projects\julia\bin\julia -e "versioninfo(); Pkg.add(\"PyCall\")" + - "%PYTHONDIR%\\python.exe -m pip install --quiet tox" test_script: - - "SET PATH=%PYTHONDIR%;%PYTHONDIR%\\Scripts;C:\\projects\\julia\\bin;%PATH%" - - python --version + - "SET PATH=%cd%\\%BATDIR%;%PYTHONDIR%;%PYTHONDIR%\\Scripts;C:\\projects\\julia\\bin;%PATH%" - dir - - ps: if (Test-Path Env:\CROSS_VERSION_PATH) { Invoke-Expression "$env:CROSS_VERSION_PATH -m unittest discover" } - # - python -c "import julia; julia.Julia(debug=True)" - - python -m unittest discover + + # Run cross-version tests but ignore the failures (from Python 2). + # Once cross-version in Windows is fmixed, stop using + # Invoke-Expression (which ignores the exit status). + - ps: if ($env:CROSS_VERSION -eq 1) { Invoke-Expression "tox -- -s" } + # - ps: if ($env:CROSS_VERSION -eq 1) { tox -- -s } + + # Rebuild PyCall.ji for each Python interpreter before testing: + - "SET PYJULIA_TEST_REBUILD=yes" + - tox -- -s diff --git a/ci/appveyor/win32/python2.7.bat b/ci/appveyor/win32/python2.7.bat new file mode 100644 index 00000000..0e533e5e --- /dev/null +++ b/ci/appveyor/win32/python2.7.bat @@ -0,0 +1 @@ +@C:\Python27\python.exe %* diff --git a/ci/appveyor/win32/python3.5.bat b/ci/appveyor/win32/python3.5.bat new file mode 100644 index 00000000..2074dc00 --- /dev/null +++ b/ci/appveyor/win32/python3.5.bat @@ -0,0 +1 @@ +@C:\Python35\python.exe %* diff --git a/ci/appveyor/win64/python2.7.bat b/ci/appveyor/win64/python2.7.bat new file mode 100644 index 00000000..73fb9839 --- /dev/null +++ b/ci/appveyor/win64/python2.7.bat @@ -0,0 +1 @@ +@C:\Python27-x64\python.exe %* diff --git a/ci/appveyor/win64/python3.5.bat b/ci/appveyor/win64/python3.5.bat new file mode 100644 index 00000000..2b6b3b32 --- /dev/null +++ b/ci/appveyor/win64/python3.5.bat @@ -0,0 +1 @@ +@C:\Python35-x64\python.exe %* diff --git a/ci/install-julia.sh b/ci/install-julia.sh index 023d6734..71bf1e01 100755 --- a/ci/install-julia.sh +++ b/ci/install-julia.sh @@ -1,30 +1,23 @@ -#!/bin/sh -# install julia release: ./install-julia.sh juliareleases -# install julia nightly: ./install-julia.sh julianightlies - -VERSION="0.6.2" -SHORTVERSION="0.6" +#!/bin/bash +# install julia vX.Y.Z: ./install-julia.sh X.Y.Z +# install julia nightly: ./install-julia.sh nightly # stop on error set -e -# default to juliareleases -if [ $# -ge 1 ]; then - JULIAVERSION=$1 -elif [ -z "$JULIAVERSION" ]; then - JULIAVERSION=juliareleases -fi +VERSION="$1" -case "$JULIAVERSION" in - julianightlies) +case "$VERSION" in + nightly) BASEURL="https://julialangnightlies-s3.julialang.org/bin" JULIANAME="julia-latest" ;; - juliareleases) + [0-9]*) BASEURL="https://julialang-s3.julialang.org/bin" + SHORTVERSION="$(echo "$VERSION" | grep -Eo '^[0-9]+\.[0-9]+')" JULIANAME="$SHORTVERSION/julia-$VERSION" ;; *) - echo "Unrecognized JULIAVERSION=$JULIAVERSION, exiting" + echo "Unrecognized VERSION=$VERSION, exiting" exit 1 ;; esac @@ -34,22 +27,22 @@ case $(uname) in case $(uname -m) in x86_64) ARCH="x64" - case "$JULIAVERSION" in - julianightlies) + case "$JULIANAME" in + julia-latest) SUFFIX="linux64" ;; - juliareleases) + *) SUFFIX="linux-x86_64" ;; esac ;; i386 | i486 | i586 | i686) ARCH="x86" - case "$JULIAVERSION" in - julianightlies) + case "$JULIANAME" in + julia-latest) SUFFIX="linux32" ;; - juliareleases) + *) SUFFIX="linux-i686" ;; esac diff --git a/julia/core.py b/julia/core.py index 160372ec..959d8d96 100644 --- a/julia/core.py +++ b/julia/core.py @@ -24,6 +24,7 @@ import time import warnings +from collections import namedtuple from ctypes import c_void_p as void_p from ctypes import c_char_p as char_p from ctypes import py_object @@ -141,6 +142,10 @@ class JuliaImporter(object): # find_module was deprecated in v3.4 def find_module(self, fullname, path=None): if fullname.startswith("julia."): + pypath = os.path.join(os.path.dirname(__file__), + "{}.py".format(fullname[len("julia."):])) + if os.path.isfile(pypath): + return return JuliaModuleLoader() @@ -247,6 +252,44 @@ def determine_if_statically_linked(): return not (b"libpython" in lddoutput) +JuliaInfo = namedtuple( + 'JuliaInfo', + ['JULIA_HOME', 'libjulia_path', 'image_file', 'pyprogramname']) + + +def juliainfo(runtime='julia'): + output = subprocess.check_output( + [runtime, "-e", + """ + println(VERSION < v"0.7.0-DEV.3073" ? JULIA_HOME : Base.Sys.BINDIR) + println(Libdl.dlpath(string("lib", splitext(Base.julia_exename())[1]))) + println(unsafe_string(Base.JLOptions().image_file)) + PyCall_depsfile = Pkg.dir("PyCall","deps","deps.jl") + if isfile(PyCall_depsfile) + eval(Module(:__anon__), + Expr(:toplevel, + :(Main.Base.include($PyCall_depsfile)), + :(println(pyprogramname)))) + end + """]) + args = output.decode("utf-8").rstrip().split("\n") + if len(args) == 3: + args.append(None) # no pyprogramname set + return JuliaInfo(*args) + + +def is_same_path(a, b): + a = os.path.normpath(os.path.normcase(a)) + b = os.path.normpath(os.path.normcase(b)) + return a == b + + +def is_different_exe(pyprogramname, sys_executable): + if pyprogramname is None: + return True + return not is_same_path(pyprogramname, sys_executable) + + _julia_runtime = [False] class Julia(object): @@ -292,29 +335,18 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, self.api = _julia_runtime[0] return + self._debug() # so that debug message is shown nicely w/ pytest + if init_julia: if jl_runtime_path: runtime = jl_runtime_path else: runtime = 'julia' - juliainfo = subprocess.check_output( - [runtime, "-e", - """ - println(VERSION < v"0.7.0-DEV.3073" ? JULIA_HOME : Base.Sys.BINDIR) - println(Libdl.dlpath(string("lib", splitext(Base.julia_exename())[1]))) - println(unsafe_string(Base.JLOptions().image_file)) - PyCall_depsfile = Pkg.dir("PyCall","deps","deps.jl") - if isfile(PyCall_depsfile) - eval(Module(:__anon__), - Expr(:toplevel, - :(Main.Base.include($PyCall_depsfile)), - :(println(pyprogramname)))) - else - println("nowhere") - end - """]) - JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo.decode("utf-8").rstrip().split("\n") - exe_differs = not depsjlexe == sys.executable + JULIA_HOME, libjulia_path, image_file, depsjlexe = juliainfo() + self._debug("pyprogramname =", depsjlexe) + self._debug("sys.executable =", sys.executable) + exe_differs = is_different_exe(depsjlexe, sys.executable) + self._debug("exe_differs =", exe_differs) self._debug("JULIA_HOME = %s, libjulia_path = %s" % (JULIA_HOME, libjulia_path)) if not os.path.exists(libjulia_path): raise JuliaError("Julia library (\"libjulia\") not found! {}".format(libjulia_path)) @@ -432,12 +464,13 @@ def __init__(self, init_julia=True, jl_runtime_path=None, jl_init_path=None, self.sprint = self.eval('sprint') self.showerror = self.eval('showerror') - def _debug(self, msg): + def _debug(self, *msg): """ Print some debugging stuff, if enabled """ if self.is_debugging: - print(msg, file=sys.stderr) + print(*msg, file=sys.stderr) + sys.stderr.flush() def _call(self, src): """ diff --git a/julia/magic.py b/julia/magic.py index f53699fe..96b3d7ae 100644 --- a/julia/magic.py +++ b/julia/magic.py @@ -17,7 +17,7 @@ # Imports #----------------------------------------------------------------------------- -from __future__ import print_function +from __future__ import print_function, absolute_import import sys from IPython.core.magic import Magics, magics_class, line_cell_magic @@ -46,7 +46,7 @@ def __init__(self, shell): end='') # Flush, otherwise the Julia startup will keep stdout buffered sys.stdout.flush() - self.julia = Julia(init_julia=True) + self._julia = Julia(init_julia=True) print() @line_cell_magic @@ -58,9 +58,9 @@ def julia(self, line, cell=None): src = compat.unicode_type(line if cell is None else cell) try: - ans = self.julia.eval(src) + ans = self._julia.eval(src) except JuliaError as e: - print(e.message, file=sys.stderr) + print(e, file=sys.stderr) ans = None return ans diff --git a/julia/with_rebuilt.py b/julia/with_rebuilt.py new file mode 100644 index 00000000..7c4b7cac --- /dev/null +++ b/julia/with_rebuilt.py @@ -0,0 +1,76 @@ +""" +(Maybe) Re-build PyCall.jl to test ``exe_differs=False`` path. + +``Pkg.build("PyCall")`` is run on Julia side when the environment +variable `PYJULIA_TEST_REBUILD` is set to ``yes``. +""" + +from __future__ import print_function, absolute_import + +import os +import subprocess +import sys +from contextlib import contextmanager + +from .core import juliainfo + + +@contextmanager +def maybe_rebuild(rebuild, julia): + if rebuild: + env = os.environ.copy() + info = juliainfo(julia) + + build = [julia, '-e', 'Pkg.build("PyCall")'] + print('Building PyCall.jl with PYTHON =', sys.executable) + print(*build) + sys.stdout.flush() + subprocess.check_call(build, env=dict(env, PYTHON=sys.executable)) + try: + yield + finally: + print('Restoring previous PyCall.jl build...') + print(*build) + if info.pyprogramname: + # Use str to avoid "TypeError: environment can only + # contain strings" in Python 2.7 + Windows: + env = dict(env, PYTHON=str(info.pyprogramname)) + if 'PYTHON' in env: + print('PYTHON =', env['PYTHON']) + subprocess.check_call(build, env=env) + else: + yield + + +def with_rebuilt(rebuild, julia, command): + with maybe_rebuild(rebuild, julia): + print('Execute:', *command) + return subprocess.call(command) + + +def main(args=None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description=__doc__) + parser.add_argument( + '--rebuild', default=os.getenv('PYJULIA_TEST_REBUILD', 'no'), + choices=('yes', 'no'), + help=""" + """) + parser.add_argument( + '--julia', default=os.getenv('JULIA_EXE', 'julia'), + help=""" + Julia executable to be used. + Default to the value of environment variable JULIA_EXE if set. + """) + parser.add_argument( + 'command', nargs='+', + help='Command and arguments to run.') + ns = parser.parse_args(args) + ns.rebuild = ns.rebuild == 'yes' + sys.exit(with_rebuilt(**vars(ns))) + + +if __name__ == '__main__': + main() diff --git a/test/test_core.py b/test/test_core.py index 4391901d..ed998d9b 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -4,6 +4,7 @@ import math import subprocess import unittest +from contextlib import contextmanager from types import ModuleType from julia import Julia, JuliaError @@ -17,6 +18,7 @@ orig_env = os.environ.copy() julia = Julia(jl_runtime_path=os.getenv("JULIA_EXE"), debug=True) + class JuliaTest(unittest.TestCase): def test_call(self): diff --git a/test/test_magic.py b/test/test_magic.py new file mode 100644 index 00000000..70344d4c --- /dev/null +++ b/test/test_magic.py @@ -0,0 +1,34 @@ +from IPython.testing.globalipapp import get_ipython +import julia.magic + + +def get_julia_magics(): + return julia.magic.JuliaMagics(shell=get_ipython()) + + +def test_register_magics(): + julia.magic.load_ipython_extension(get_ipython()) + + +def test_success_line(): + jm = get_julia_magics() + ans = jm.julia('1') + assert ans == 1 + + +def test_success_cell(): + jm = get_julia_magics() + ans = jm.julia(None, '2') + assert ans == 2 + + +def test_failure_line(): + jm = get_julia_magics() + ans = jm.julia('pop!([])') + assert ans is None + + +def test_failure_cell(): + jm = get_julia_magics() + ans = jm.julia(None, '1 += 1') + assert ans is None diff --git a/test/test_utils.py b/test/test_utils.py new file mode 100644 index 00000000..a86d146b --- /dev/null +++ b/test/test_utils.py @@ -0,0 +1,18 @@ +""" +Unit tests which can be done without loading `libjulia`. +""" + +import sys + +import pytest + +from julia.core import is_different_exe + + +@pytest.mark.parametrize('pyprogramname, sys_executable, exe_differs', [ + (sys.executable, sys.executable, False), + (None, sys.executable, True), + ('/dev/null', sys.executable, True), +]) +def test_is_different_exe(pyprogramname, sys_executable, exe_differs): + assert is_different_exe(pyprogramname, sys_executable) == exe_differs diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..0ddbb827 --- /dev/null +++ b/tox.ini @@ -0,0 +1,24 @@ +[tox] +envlist = py27, py36 + +[testenv] +deps = + pytest + numpy + ipython + mock +commands = + python -m julia.with_rebuilt -- python -m pytest {posargs} + # Using "python -m pytest" to exactly match the Python interpreter + # used to build PyCall.jl via julia/with_rebuilt.py (when + # PYJULIA_TEST_REBUILD=yes). + +passenv = + # Allow a workaround for "error initializing LibGit2 module": + # https://github.com/JuliaLang/julia/issues/18693 + # https://github.com/JuliaDiffEq/diffeqpy/pull/13/commits/850441ee63962a2417de2bce6f6223052ee9cceb + SSL_CERT_FILE + + # See: julia/with_rebuilt.py + PYJULIA_TEST_REBUILD + JULIA_EXE