diff --git a/jose/backends/__init__.py b/jose/backends/__init__.py
index 732bd5b2..d1b9fa1a 100644
--- a/jose/backends/__init__.py
+++ b/jose/backends/__init__.py
@@ -1,9 +1,9 @@
 
 try:
-    from jose.backends.pycrypto_backend import RSAKey  # noqa: F401
+    from jose.backends.cryptography_backend import CryptographyRSAKey as RSAKey  # noqa: F401
 except ImportError:
     try:
-        from jose.backends.cryptography_backend import CryptographyRSAKey as RSAKey  # noqa: F401
+        from jose.backends.pycrypto_backend import RSAKey  # noqa: F401
     except ImportError:
         from jose.backends.rsa_backend import RSAKey  # noqa: F401
 
diff --git a/tests/algorithms/test_RSA.py b/tests/algorithms/test_RSA.py
index 944b9945..20a59422 100644
--- a/tests/algorithms/test_RSA.py
+++ b/tests/algorithms/test_RSA.py
@@ -345,8 +345,9 @@ def test_pycrypto_RSA_key_instance():
 @pytest.mark.pycrypto
 @pytest.mark.pycryptodome
 @pytest.mark.parametrize("private_key", PRIVATE_KEYS)
+@pytest.mark.skipif(None in (PyCryptoRSA, PyCryptoRSAKey), reason="Pycrypto/dome backend not available")
 def test_pycrypto_unencoded_cleartext(private_key):
-    key = RSAKey(private_key, ALGORITHMS.RS256)
+    key = PyCryptoRSAKey(private_key, ALGORITHMS.RS256)
     msg = b'test'
     signature = key.sign(msg)
     public_key = key.public_key()
diff --git a/tests/test_backends.py b/tests/test_backends.py
new file mode 100644
index 00000000..6e633a19
--- /dev/null
+++ b/tests/test_backends.py
@@ -0,0 +1,35 @@
+"""Test the default import handling."""
+try:
+    from jose.backends.rsa_backend import RSAKey as PurePythonRSAKey
+except ImportError:
+    PurePythonRSAKey = None
+try:
+    from jose.backends.cryptography_backend import CryptographyRSAKey, CryptographyECKey
+except ImportError:
+    CryptographyRSAKey = CryptographyECKey = None
+try:
+    from jose.backends.pycrypto_backend import RSAKey as PyCryptoRSAKey
+except ImportError:
+    PyCryptoRSAKey = None
+try:
+    from jose.backends.ecdsa_backend import ECDSAECKey as PurePythonECDSAKey
+except ImportError:
+    PurePythonRSAKey = None
+
+from jose.backends import ECKey, RSAKey
+
+
+def test_default_ec_backend():
+    if CryptographyECKey is not None:
+        assert ECKey is CryptographyECKey
+    else:
+        assert ECKey is PurePythonECDSAKey
+
+
+def test_default_rsa_backend():
+    if CryptographyRSAKey is not None:
+        assert RSAKey is CryptographyRSAKey
+    elif PyCryptoRSAKey is not None:
+        assert RSAKey is PyCryptoRSAKey
+    else:
+        assert RSAKey is PurePythonRSAKey