Improve pytest plugin and its documentation

docs/source/pytest.rst

@@ -13,15 +13,31 @@ tricky aspects of PyJulia initialization:
   bundled with ``julia`` are newer than the ones otherwise loaded).
 
 * It provides a way to succinctly mark certain tests require Julia
-  runtime (see `Fixture`_).
+  runtime (see `Fixture`_ and `Marker`_).
 
 * The tests requiring Julia can be skipped with :option:`--no-julia`.
 
-Use |pytest -p no:julia|_ to disable PyJulia plugin.
+* It enables debug-level logging.  This is highly recommended
+  especially in CI setting as miss-configuration of PyJulia may result
+  in segmentation fault in which Python cannot provide useful
+  traceback.
 
-.. |pytest -p no:julia| replace:: ``pytest -p no:julia``
-.. _pytest -p no:julia:
-   https://docs.pytest.org/en/latest/plugins.html#deactivating-unregistering-a-plugin-by-name
+To activate PyJulia's pytest plugin [#]_ add ``-p julia.pytestplugin``
+to the command line option.  There are several ways to do this by
+default in your project.  One option is to include this using
+``addopts`` setup of ``pytest.ini`` or ``tox.ini`` file.  See `How to
+change command line options defaults
+<https://docs.pytest.org/en/latest/customize.html#adding-default-options>`_:
+
+.. code-block:: ini
+
+   [pytest]
+   addopts =
+       -p julia.pytestplugin
+
+.. [#] This plugin is not activated by default (as in normal
+   ``pytest-*`` plugin packages) to avoid accidentally breaking user's
+   ``pytest`` setup when PyJulia is included as a non-test dependency.
 
 
 Options
@@ -38,6 +54,15 @@ Following options can be passed to :program:`pytest`
    Julia executable to be used.  Defaults to environment variable
    `PYJULIA_TEST_RUNTIME`.
 
+.. option:: --julia-<julia_option>
+
+   Some ``<julia_option>`` that can be passed to ``julia`` executable
+   (e.g., ``--compiled-modules=no``) can be passed to ``pytest``
+   plugin by ``--julia-<julia_option>`` (e.g.,
+   ``--julia-compiled-modules=no``).  See ``pytest -p
+   julia.pytestplugin --help`` for the actual list of options.
+
+
 Fixture
 =======
 
@@ -52,3 +77,21 @@ initialized.  Example usage::
 This fixture also "marks" that this test requires a Julia runtime.
 Thus, the tests using ``julia`` fixture are not run when
 :option:`--no-julia` is passed.
+
+
+Marker
+======
+
+PyJulia's pytest plugin also includes a `pytest marker
+<https://docs.pytest.org/en/latest/example/markers.html>`_ ``julia``
+which can be used to mark that the test requires PyJulia setup.  It is
+similar to ``julia`` fixture but it does not instantiate the actual
+:class:`.Julia` object.
+
+Example usage::
+
+   import pytest
+
+   @pytest.mark.julia
+   def test_import():
+       from julia import MyModule

setup.py

@@ -71,9 +71,6 @@ def pyload(path):
           "console_scripts": [
               "python-jl = julia.python_jl:main",
           ],
-          "pytest11": [
-              "julia = julia.pytestplugin",
-          ],
       },
       # We bundle Julia scripts etc. inside `julia` directory.  Thus,
       # this directory must exist in the file system (not in a zip

src/julia/options.py

@@ -13,6 +13,14 @@ def __get__(self, instance, owner):
             return self
         return getattr(instance, self.dataname, self.default)
 
+    def cli_argument_name(self):
+        name = {"bindir": "home"}.get(self.name, self.name)
+        return "--" + name.replace("_", "-")
+
+    def cli_argument_spec(self):
+        return dict(help="Julia's ``{}`` option.".format(self.cli_argument_name()))
+        # TODO: parse help from `options_docs`.
+
 
 class String(OptionDescriptor):
     def __init__(self, name, default=None):
@@ -54,6 +62,12 @@ def __set__(self, instance, value):
     def _domain(self):  # used in test
         return set(self.choicemap)
 
+    def cli_argument_spec(self):
+        return dict(
+            super(Choices, self).cli_argument_spec(),
+            choices=list(self.choicemap.values()),
+        )
+
 
 def yes_no_etc(*etc):
     choicemap = {True: "yes", False: "no", "yes": "yes", "no": "no"}
@@ -149,16 +163,21 @@ def is_specified(self, name):
     def specified(self):
         for name in dir(self.__class__):
             if self.is_specified(name):
-                yield name, getattr(self, name)
+                yield getattr(self.__class__, name), getattr(self, name)
 
     def as_args(self):
         args = []
-        for (name, value) in self.specified():
-            name = {"bindir": "home"}.get(name, name)
-            args.append("--" + name.replace("_", "-"))
+        for (desc, value) in self.specified():
+            args.append(desc.cli_argument_name())
             args.append(value)
         return args
 
+    @classmethod
+    def supported_options(cls):
+        for name in dir(cls):
+            if cls.is_supported(name):
+                yield getattr(cls, name)
+
 
 def parse_jl_options(options):
     """

src/julia/pytestplugin.py

@@ -2,12 +2,16 @@
 
 import pytest
 
+from .options import JuliaOptions
+
+_USING_DEFAULT_SETUP = True
+
 
 def pytest_addoption(parser):
     import os
 
     # Note: the help strings have to be synchronized manually with
-    # ../docs/source/pytest.rst
+    # ../../docs/source/pytest.rst
 
     parser.addoption(
         "--no-julia",
@@ -24,15 +28,39 @@ def pytest_addoption(parser):
         default=os.getenv("PYJULIA_TEST_RUNTIME", "julia"),
     )
 
+    for desc in JuliaOptions.supported_options():
+        parser.addoption(
+            "--julia-{}".format(desc.cli_argument_name().lstrip("-")),
+            **desc.cli_argument_spec()
+        )
+
 
 def pytest_sessionstart(session):
     if not session.config.getoption("julia"):
         return
 
-    from .core import Julia
+    from .core import LibJulia, JuliaInfo, Julia, enable_debug
+
+    options = JuliaOptions()
+    for desc in JuliaOptions.supported_options():
+        cli_option = "--julia-{}".format(desc.cli_argument_name().lstrip("-"))
+        desc.__set__(options, session.config.getoption(cli_option))
+
+    julia_runtime = session.config.getoption("julia_runtime")
+
+    global _USING_DEFAULT_SETUP
+    _USING_DEFAULT_SETUP = not (julia_runtime or options.as_args())
+
+    enable_debug()
+    info = JuliaInfo.load(julia=julia_runtime)
+    if (info.version_major, info.version_minor) < (0, 7):
+        # In Julia 0.6, we have to load PyCall.jl here to do the
+        # fake-julia setup.
+        Julia(runtime=julia_runtime)
+    else:
+        api = LibJulia.from_juliainfo(info)
+        api.init_julia(options)
 
-    global JULIA
-    JULIA = Julia(runtime=session.config.getoption("julia_runtime"), debug=True)
 
 # Initialize Julia runtime as soon as possible (or more precisely
 # before importing any additional Python modules) to avoid, e.g.,
@@ -48,4 +76,18 @@ def julia(request):
     if not request.config.getoption("julia"):
         pytest.skip("--no-julia is given.")
 
-    return JULIA
+    from .core import Julia
+
+    return Julia()
+
+
+def pytest_runtest_setup(item):
+    if not item.config.getoption("julia"):
+        for mark in item.iter_markers("julia"):
+            pytest.skip("--no-julia is given.")
+
+    if not (item.config.getoption("julia") and _USING_DEFAULT_SETUP):
+        for mark in item.iter_markers("pyjulia__using_default_setup"):
+            pytest.skip(
+                "using non-default setup (e.g., --julia-<julia_option> is given)"
+            )

test/test_core.py

@@ -147,15 +147,7 @@ def test_module_dir(julia):
     assert "resize_b" in dir(Base)
 
 
-@pytest.mark.skipif(
-    "PYJULIA_TEST_RUNTIME" in orig_env,
-    reason=(
-        "cannot be tested with custom Julia executable;"
-        " PYJULIA_TEST_RUNTIME is set to {}".format(
-            orig_env.get("PYJULIA_TEST_RUNTIME")
-        )
-    ),
-)
+@pytest.mark.pyjulia__using_default_setup
 def test_import_without_setup():
     command = [sys.executable, "-c", "from julia import Base"]
     print("Executing:", *command)

test/test_libjulia.py

@@ -9,6 +9,7 @@
 
 
 @pytest.mark.skipif("juliainfo.version_info < (0, 7)")
+@pytest.mark.julia
 def test_compiled_modules_no():
     runcode(
         sys.executable,
@@ -28,6 +29,7 @@ def test_compiled_modules_no():
 
 
 @pytest.mark.skipif("not juliainfo.is_compatible_python()")
+@pytest.mark.julia
 def test_custom_sysimage(tmpdir):
     sysimage = str(tmpdir.join("sys.so"))
     runcode(
@@ -58,6 +60,7 @@ def test_custom_sysimage(tmpdir):
     )
 
 
+@pytest.mark.julia
 def test_non_existing_sysimage(tmpdir):
     proc = runcode(
         sys.executable,

test/test_options.py

@@ -13,20 +13,13 @@ def parse_options_docs(docs):
     return optdefs
 
 
-def supported_names(cls):
-    for name in dir(cls):
-        if cls.is_supported(name):
-            yield name
-
-
 def test_options_docs():
     """
     Ensure that `JuliaOptions` and `JuliaOptions.__doc__` agree.
     """
 
     optdefs = parse_options_docs(options_docs)
-    for name in supported_names(JuliaOptions):
-        odef = optdefs.pop(name)
-        desc = getattr(JuliaOptions, name)
+    for desc in JuliaOptions.supported_options():
+        odef = optdefs.pop(desc.name)
         assert odef["domain"] == desc._domain()
     assert not optdefs

test/test_python_jl.py

@@ -60,6 +60,7 @@ def test_cli_quick_pass_no_julia(args):
 @pytest.mark.skipif(
     not PYJULIA_TEST_REBUILD,
     reason="PYJULIA_TEST_REBUILD=yes is not set")
+@pytest.mark.julia
 def test_cli_import():
     args = ["-c", dedent("""
     from julia import Base

test/test_utils.py

@@ -44,6 +44,7 @@ def test_raise_separate_cache_error_dynamically_linked():
     assert "have to match exactly" in str(excinfo.value)
 
 
+@pytest.mark.pyjulia__using_default_setup
 def test_atexit():
     proc = runcode(
         sys.executable,

tox.ini

@@ -43,6 +43,7 @@ passenv =
 
 [pytest]
 addopts =
+    -p julia.pytestplugin
     --doctest-modules
     --ignore=test/_star_import.py