diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
index 2879d37cb405c0..75b83fbe8bae61 100644
--- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
+++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs
@@ -1979,7 +1979,12 @@ public bool CleanCacheAndDisposeIfUnused()
             }
 
             // Dispose the stale connections outside the pool lock, to avoid holding the lock too long.
-            toDispose?.ForEach(c => c.Dispose());
+            // Dispose them asynchronously to not to block the caller on closing the SslStream or NetworkStream.
+            if (toDispose is not null)
+            {
+                Task.Factory.StartNew(static s => ((List<HttpConnectionBase>)s!).ForEach(c => c.Dispose()), toDispose,
+                    CancellationToken.None, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default);
+            }
 
             // Pool is active.  Should not be removed.
             return false;
diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
index e257cb4c3e692f..f118d151c692ae 100644
--- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
+++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPoolManager.cs
@@ -462,7 +462,7 @@ private void SetCleaningTimer(TimeSpan timeout)
         {
             try
             {
-                _cleaningTimer!.Change(timeout, timeout);
+                _cleaningTimer!.Change(timeout, Timeout.InfiniteTimeSpan);
                 _timerIsRunning = timeout != Timeout.InfiniteTimeSpan;
             }
             catch (ObjectDisposedException)
@@ -492,13 +492,10 @@ private void RemoveStalePools()
                 }
             }
 
-            // Stop running the timer if we don't have any pools to clean up.
+            // Restart the timer if we have any pools to clean up.
             lock (SyncObj)
             {
-                if (_pools.IsEmpty)
-                {
-                    SetCleaningTimer(Timeout.InfiniteTimeSpan);
-                }
+                SetCleaningTimer(!_pools.IsEmpty ? _cleanPoolTimeout : Timeout.InfiniteTimeSpan);
             }
 
             // NOTE: There is a possible race condition with regards to a pool getting cleaned up at the same