diff --git a/jose/backends/pycrypto_backend.py b/jose/backends/pycrypto_backend.py
index a12e861c..42abe2a1 100644
--- a/jose/backends/pycrypto_backend.py
+++ b/jose/backends/pycrypto_backend.py
@@ -147,6 +147,8 @@ def sign(self, msg):
             raise JWKError(e)
 
     def verify(self, msg, sig):
+        if not self.is_public():
+            return False
         try:
             return PKCS1_v1_5.new(self.prepared_key).verify(self.hash_alg.new(msg), sig)
         except Exception:
diff --git a/jose/backends/rsa_backend.py b/jose/backends/rsa_backend.py
index c1f5539d..82423a79 100644
--- a/jose/backends/rsa_backend.py
+++ b/jose/backends/rsa_backend.py
@@ -200,6 +200,8 @@ def sign(self, msg):
         return pyrsa.sign(msg, self._prepared_key, self.hash_alg)
 
     def verify(self, msg, sig):
+        if not self.is_public():
+            return False
         try:
             pyrsa.verify(msg, sig, self._prepared_key)
             return True
diff --git a/tests/test_jws.py b/tests/test_jws.py
index f543a03a..c087e850 100644
--- a/tests/test_jws.py
+++ b/tests/test_jws.py
@@ -291,6 +291,16 @@ def test_wrong_key(self, payload):
         with pytest.raises(JWSError):
             jws.verify(token, rsa_public_key, ALGORITHMS.HS256)
 
+    def test_private_verify(self, payload):
+        token = jws.sign(payload, rsa_private_key, algorithm='RS256')
+
+        # verify with public
+        dec = jws.verify(token, rsa_public_key, algorithms='RS256')
+
+        with pytest.raises(JWSError):
+            # verify with private does not work
+            dec = jws.verify(token, rsa_private_key, algorithms='RS256')
+
 
 ec_private_key = """-----BEGIN EC PRIVATE KEY-----
 MIHcAgEBBEIBzs13YUnYbLfYXTz4SG4DE4rPmsL3wBTdy34JcO+BDpI+NDZ0pqam