Use find_libpython.py in deps/build.jl (#556)

* Copy find_libpython.py from PyJulia

* Call find_libpython.py from deps/build.jl

* Handle the case find_libpython.py cannot find libpython

* Store full path to libpython in deps.jl

* Update find_libpython.py

* Use --candidate-names in the fallback path

* Improve debugging message

* Use pythonenv in when invoking find_libpython.py

* Add _linked_libpython_windows

Author: tkf

deps/build.jl

@@ -47,91 +47,65 @@ pysys(python::AbstractString, var::AbstractString) = pyvar(python, "sys", var)
 
 const dlprefix = Compat.Sys.iswindows() ? "" : "lib"
 
-# return libpython name, libpython pointer
-function find_libpython(python::AbstractString)
-    # it is ridiculous that it is this hard to find the name of libpython
-    v = pyconfigvar(python,"VERSION","")
-    libs = [ dlprefix*"python"*v, dlprefix*"python" ]
-    lib = pyconfigvar(python, "LIBRARY")
-    lib != "None" && pushfirst!(libs, splitext(lib)[1])
-    lib = pyconfigvar(python, "LDLIBRARY")
-    lib != "None" && pushfirst!(pushfirst!(libs, basename(lib)), lib)
-    libs = unique(libs)
-
-    # it is ridiculous that it is this hard to find the path of libpython
-    libpaths = [pyconfigvar(python, "LIBDIR"),
-                (Compat.Sys.iswindows() ? dirname(pysys(python, "executable")) : joinpath(dirname(dirname(pysys(python, "executable"))), "lib"))]
-    if Compat.Sys.isapple()
-        push!(libpaths, pyconfigvar(python, "PYTHONFRAMEWORKPREFIX"))
-    end
+# print out extra info to help with remote debugging
+const PYCALL_DEBUG_BUILD = "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no")
 
-    # `prefix` and `exec_prefix` are the path prefixes where python should look for python only and compiled libraries, respectively.
-    # These are also changed when run in a virtualenv.
-    exec_prefix = pysys(python, "exec_prefix")
-
-    push!(libpaths, exec_prefix)
-    push!(libpaths, joinpath(exec_prefix, "lib"))
+function exec_find_libpython(python::AbstractString, options)
+    cmd = `$python $(joinpath(@__DIR__, "find_libpython.py")) $options`
+    if PYCALL_DEBUG_BUILD
+        cmd = `$cmd --verbose`
+    end
+    return readlines(pythonenv(cmd))
+end
 
-    error_strings = String[]
+function show_dlopen_error(e)
+    if PYCALL_DEBUG_BUILD
+        println(stderr, "dlopen($libpath_lib) ==> ", e)
+        # Using STDERR since find_libpython.py prints debugging
+        # messages to STDERR too.
+    end
+end
 
-    # TODO: other paths? python-config output? pyconfigvar("LDFLAGS")?
+# return libpython name, libpython pointer
+function find_libpython(python::AbstractString)
+    dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL
 
-    # find libpython (we hope):
-    for lib in libs
-        for libpath in libpaths
-            libpath_lib = joinpath(libpath, lib)
-            if isfile(libpath_lib*"."*Libdl.dlext)
-                try
-                    return (Libdl.dlopen(libpath_lib,
-                                         Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL),
-                            libpath_lib)
-                catch e
-                    push!(error_strings, string("dlopen($libpath_lib) ==> ", e))
-                end
-            end
+    libpaths = exec_find_libpython(python, `--list-all`)
+    for lib in libpaths
+        try
+            return (Libdl.dlopen(lib, dlopen_flags), lib)
+        catch e
+            show_dlopen_error(e)
         end
     end
 
+    # Try all candidate libpython names and let Libdl find the path.
     # We do this *last* because the libpython in the system
     # library path might be the wrong one if multiple python
     # versions are installed (we prefer the one in LIBDIR):
+    libs = exec_find_libpython(python, `--candidate-names`)
     for lib in libs
         lib = splitext(lib)[1]
         try
-            return (Libdl.dlopen(lib, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL),
-                    lib)
+            libpython = Libdl.dlopen(lib, dlopen_flags)
+            # Store the fullpath to libpython in deps.jl.  This makes
+            # it easier for users to investigate Python setup
+            # PyCall.jl trying to use.  It also helps PyJulia to
+            # compare libpython.
+            return (libpython, Libdl.dlpath(libpython))
         catch e
-            push!(error_strings, string("dlopen($lib) ==> ", e))
-        end
-    end
-
-    if "yes" == get(ENV, "PYCALL_DEBUG_BUILD", "no") # print out extra info to help with remote debugging
-        println(stderr, "------------------------------------- exceptions -----------------------------------------")
-        for s in error_strings
-            print(s, "\n\n")
-        end
-        println(stderr, "---------------------------------- get_config_vars ---------------------------------------")
-        print(stderr, read(`python -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_vars())"`, String))
-        println(stderr, "--------------------------------- directory contents -------------------------------------")
-        for libpath in libpaths
-            if isdir(libpath)
-                print(libpath, ":\n")
-                for file in readdir(libpath)
-                    if occursin("pyth", file)
-                        println("    ", file)
-                    end
-                end
-            end
+            show_dlopen_error(e)
         end
-        println(stderr, "------------------------------------------------------------------------------------------")
     end
 
     error("""
         Couldn't find libpython; check your PYTHON environment variable.
 
-        The python executable we tried was $python (= version $v);
-        the library names we tried were $libs
-        and the library paths we tried were $libpaths""")
+        The python executable we tried was $python (= version $v).
+        Re-building with
+            ENV["PYCALL_DEBUG_BUILD"] = "yes"
+        may provide extra information for why it failed.
+        """)
 end
 
 #########################################################################