merge logic in one func

Author: Ted-Jiang

datafusion/src/datasource/object_store/mod.rs

@@ -141,7 +141,7 @@ pub trait ObjectStore: Sync + Send + Debug {
         let suffix = suffix.to_owned();
         Ok(Box::pin(file_stream.filter(move |fr| {
             let has_suffix = match fr {
-                Ok(f) => f.path().ends_with(&suffix) && !should_filter_out(f.path()),
+                Ok(f) => do_path_filter_with_suffix(f.path(), &suffix),
                 Err(_) => true,
             };
             async move { has_suffix }
@@ -161,12 +161,13 @@ pub trait ObjectStore: Sync + Send + Debug {
 }
 /// Checks if we should filter out this path name
 /// https://github.com/apache/spark/blob/5b91381b9bcea27b7be6d9cef852efbd6c23d98a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala#L358
-fn should_filter_out(path: &str) -> bool {
+fn do_path_filter_with_suffix(path: &str, suffix: &str) -> bool {
     let exclude = path.starts_with('_') && !path.contains('=')
         || path.starts_with('.')
         || path.ends_with("._COPYING_");
     let include = path.starts_with("_common_metadata") || path.starts_with("_metadata");
-    exclude && !include
+    let is_suffix = path.ends_with(suffix);
+    (!exclude || include) && is_suffix
 }
 
 static LOCAL_SCHEME: &str = "file";