Revert "source and test changes"

This reverts commit 12e6bff.

Author: virajjasani

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -3769,7 +3769,8 @@ public boolean mkdirs(Path p, FsPermission permission) throws IOException,
             createStoreContext(),
             path,
             createMkdirOperationCallbacks(),
-            performanceCreation || isMagicCommitPath(path)));
+            isMagicCommitPath(path),
+            performanceCreation));
   }
 
   /**
@@ -4208,6 +4209,7 @@ public boolean createEmptyDir(Path path, StoreContext storeContext)
               storeContext,
               path,
               createMkdirOperationCallbacks(),
+              false,
               performanceCreation));
     }
   }

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/MkdirOperation.java

@@ -69,23 +69,28 @@ public class MkdirOperation extends ExecutingStoreOperation<Boolean> {
 
   private final boolean performanceCreation;
 
+  private final boolean isMagicPath;
+
   /**
    * Initialize Mkdir Operation context for S3A.
    *
    * @param storeContext Store context.
    * @param dir Dir path of the directory.
    * @param callbacks MkdirCallbacks object used by the Mkdir operation.
+   * @param isMagicPath True if the path is magic commit path.
    * @param performanceCreation If true, skip validation of the parent directory
    * structure.
    */
   public MkdirOperation(
       final StoreContext storeContext,
       final Path dir,
       final MkdirCallbacks callbacks,
+      final boolean isMagicPath,
       final boolean performanceCreation) {
     super(storeContext);
     this.dir = dir;
     this.callbacks = callbacks;
+    this.isMagicPath = isMagicPath;
     this.performanceCreation = performanceCreation;
   }
 
@@ -107,13 +112,14 @@ public Boolean execute() throws IOException {
     }
 
     // get the file status of the path.
-    // this is not done for magic path i.e. performanceCreation mode.
-    // For performanceCreation mode, we would probe for HEAD only.
-    // For non-performance or regular mode, the probe for both HEAD and LIST would
-    // be done.
-    S3AFileStatus fileStatus = performanceCreation
-        ? probePathStatusOrNull(dir, StatusProbeEnum.HEAD_ONLY)
-        : getPathStatusExpectingDir(dir);
+    // this is done even for a magic path, to avoid always  issuing PUT
+    // requests. Doing that without a check wouild seem to be an
+    // optimization, but it is not because
+    // 1. PUT is slower than HEAD
+    // 2. Write capacity is less than read capacity on a shard
+    // 3. It adds needless entries in versioned buckets, slowing
+    //    down subsequent operations.
+    FileStatus fileStatus = getPathStatusExpectingDir(dir);
     if (fileStatus != null) {
       if (fileStatus.isDirectory()) {
         return true;
@@ -123,6 +129,15 @@ public Boolean execute() throws IOException {
     }
     // file status was null
 
+    // is the path magic?
+    // If so, we declare success without looking any further
+    if (isMagicPath) {
+      // Create the marker file immediately,
+      // and don't delete markers
+      callbacks.createFakeDirectory(dir, true);
+      return true;
+    }
+
     // if performance creation mode is set, no need to check
     // whether the closest ancestor is dir.
     if (!performanceCreation) {
@@ -134,8 +149,7 @@ public Boolean execute() throws IOException {
 
     // Create the marker file, delete the parent entries
     // if the filesystem isn't configured to retain them
-    callbacks.createFakeDirectory(dir,
-        performanceCreation);
+    callbacks.createFakeDirectory(dir, false);
     return true;
   }
 
@@ -212,7 +226,7 @@ private S3AFileStatus getPathStatusExpectingDir(final Path path)
       throws IOException {
     S3AFileStatus status = probePathStatusOrNull(path,
         StatusProbeEnum.DIRECTORIES);
-    if (status == null && !performanceCreation) {
+    if (status == null && !isMagicPath) {
       status = probePathStatusOrNull(path,
           StatusProbeEnum.FILE);
     }