Rename Testcase for Blob endpoint

Author: bhattmanish98

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -415,7 +415,7 @@ public class AbfsConfiguration{
   private int producerQueueMaxSize;
 
   @IntegerConfigurationValidatorAnnotation(ConfigurationKey =
-          FS_AZURE_CONSUMER_MAX_LAG, DefaultValue = DEFAULT_FS_AZURE_CONSUMER_MAX_LAG)
+      FS_AZURE_CONSUMER_MAX_LAG, DefaultValue = DEFAULT_FS_AZURE_CONSUMER_MAX_LAG)
   private int listingMaxConsumptionLag;
 
   @IntegerConfigurationValidatorAnnotation(ConfigurationKey =

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java

@@ -54,6 +54,7 @@
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_CREATED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_DELETED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.ERROR_IGNORED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_COPIED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_CREATED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_DELETED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.GET_RESPONSES;
@@ -134,7 +135,8 @@ public class AbfsCountersImpl implements AbfsCounters {
       SERVER_UNAVAILABLE,
       RENAME_RECOVERY,
       METADATA_INCOMPLETE_RENAME_FAILURES,
-      RENAME_PATH_ATTEMPTS
+      RENAME_PATH_ATTEMPTS,
+      FILES_COPIED
   };
 
   private static final AbfsStatistic[] DURATION_TRACKER_LIST = {

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsStatistic.java

@@ -109,7 +109,9 @@ public enum AbfsStatistic {
       "Number of times rename operation failed due to metadata being "
           + "incomplete"),
   RENAME_PATH_ATTEMPTS("rename_path_attempts",
-      "Number of times we attempt to rename a path internally");
+      "Number of times we attempt to rename a path internally"),
+  FILES_COPIED("files_copied",
+      "Total number of files copied from the object store.");
 
   private String statName;
   private String statDescription;

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsBlobClient.java

@@ -424,7 +424,7 @@ private void fixAtomicEntriesInListResults(final AbfsRestOperation op,
     }
     List<BlobListResultEntrySchema> filteredEntries = new ArrayList<>();
     for (BlobListResultEntrySchema entry : listResultSchema.paths()) {
-      if (!takeListPathAtomicRenameKeyAction(entry.path(),
+      if (!takeListPathAtomicRenameKeyAction(entry.path(), entry.isDirectory(),
           entry.contentLength().intValue(), tracingContext)) {
         filteredEntries.add(entry);
       }
@@ -444,15 +444,14 @@ public void createNonRecursivePreCheck(Path parentPath,
       }
       getPathStatus(parentPath.toUri().getPath(), false,
           tracingContext, null);
+      incrementAbfsGetPathStatus();
     } catch (AbfsRestOperationException ex) {
       if (ex.getStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
         throw new FileNotFoundException("Cannot create file "
             + parentPath.toUri().getPath()
             + " because parent folder does not exist.");
       }
       throw ex;
-    } finally {
-      getAbfsCounters().incrementCounter(CALL_GET_FILE_STATUS, 1);
     }
   }
 
@@ -510,6 +509,7 @@ protected AbfsRestOperation createMarkerAtPath(final String path,
       final String eTag,
       final ContextEncryptionAdapter contextEncryptionAdapter,
       final TracingContext tracingContext) throws AzureBlobFileSystemException {
+    incrementAbfsCreateFile();
     return createPathRestOp(path, false, false, false, eTag,
         contextEncryptionAdapter, tracingContext);
   }
@@ -807,7 +807,6 @@ public AbfsClientRenameResult renamePath(final String source,
     BlobRenameHandler blobRenameHandler = getBlobRenameHandler(source,
         destination, sourceEtag, isAtomicRenameKey(source), tracingContext
     );
-    incrementAbfsRenamePath();
     if (blobRenameHandler.execute()) {
       final AbfsUriQueryBuilder abfsUriQueryBuilder
           = createDefaultUriQueryBuilder();
@@ -1751,6 +1750,7 @@ public void takeGetPathStatusAtomicRenameKeyAction(final Path path,
       pendingJsonFileStatus = getPathStatus(
           pendingJsonPath.toUri().getPath(), tracingContext,
           null, false);
+      incrementAbfsGetPathStatus();
       if (checkIsDir(pendingJsonFileStatus.getResult())) {
         return;
       }
@@ -1805,11 +1805,11 @@ public void takeGetPathStatusAtomicRenameKeyAction(final Path path,
    * @throws AzureBlobFileSystemException server error
    */
   private boolean takeListPathAtomicRenameKeyAction(final Path path,
-      final int renamePendingJsonLen,
+      final boolean isDirectory, final int renamePendingJsonLen,
       final TracingContext tracingContext)
       throws AzureBlobFileSystemException {
     if (path == null || path.isRoot() || !isAtomicRenameKey(
-        path.toUri().getPath()) || !path.toUri()
+        path.toUri().getPath()) || isDirectory || !path.toUri()
         .getPath()
         .endsWith(RenameAtomicity.SUFFIX)) {
       return false;
@@ -1837,7 +1837,7 @@ private boolean takeListPathAtomicRenameKeyAction(final Path path,
   }
 
   @VisibleForTesting
-  RenameAtomicity getRedoRenameAtomicity(final Path renamePendingJsonPath,
+  public RenameAtomicity getRedoRenameAtomicity(final Path renamePendingJsonPath,
       int fileLen,
       final TracingContext tracingContext) {
     return new RenameAtomicity(renamePendingJsonPath,

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

@@ -91,6 +91,12 @@
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 
 import static org.apache.commons.lang3.StringUtils.isNotEmpty;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.CALL_GET_FILE_STATUS;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_CREATED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.DIRECTORIES_DELETED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_COPIED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_CREATED;
+import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.FILES_DELETED;
 import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_PATH_ATTEMPTS;
 import static org.apache.hadoop.fs.azurebfs.AzureBlobFileSystemStore.extractEtagHeader;
 import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.APN_VERSION;
@@ -693,6 +699,34 @@ protected void incrementAbfsRenamePath() {
     abfsCounters.incrementCounter(RENAME_PATH_ATTEMPTS, 1);
   }
 
+  /**
+   * Increments AbfsCounters for get path status by 1.
+   */
+  protected void incrementAbfsGetPathStatus() {
+    abfsCounters.incrementCounter(CALL_GET_FILE_STATUS, 1);
+  }
+
+  /**
+   * Increments AbfsCounters for Delete File by 1.
+   */
+  protected void incrementAbfsDeleteFile() {
+    abfsCounters.incrementCounter(FILES_DELETED, 1);
+  }
+
+  /**
+   * Increments AbfsCounters for Create File by 1.
+   */
+  protected void incrementAbfsCreateFile() {
+    abfsCounters.incrementCounter(FILES_CREATED, 1);
+  }
+
+  /**
+   * Increments AbfsCounters for Copy Files by 1.
+   */
+  protected void incrementAbfsCopyFile() {
+    abfsCounters.incrementCounter(FILES_COPIED, 1);
+  }
+
   /**
    * Check if the rename request failure is post a retry and if earlier rename
    * request might have succeeded at back-end.

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlobDeleteHandler.java

@@ -90,6 +90,7 @@ int getMaxConsumptionParallelism() {
   private boolean deleteInternal(final Path path)
       throws AzureBlobFileSystemException {
     getAbfsClient().deleteBlobPath(path, null, tracingContext);
+    getAbfsClient().incrementAbfsDeleteFile();
     deleteCount.incrementAndGet();
     return true;
   }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/BlobRenameHandler.java

@@ -121,6 +121,7 @@ int getMaxConsumptionParallelism() {
    * @throws AzureBlobFileSystemException if server call fails
    */
   public boolean execute() throws AzureBlobFileSystemException {
+    getAbfsClient().incrementAbfsRenamePath();
     PathInformation pathInformation = getPathInformation(src, tracingContext);
     boolean result = false;
     if (preCheck(src, dst, pathInformation)) {
@@ -439,7 +440,9 @@ private boolean renameInternal(final Path path,
     boolean operated = false;
     try {
       copyPath(path, destinationPathForBlobPartOfRenameSrcDir, leaseId);
+      getAbfsClient().incrementAbfsCopyFile();
       getAbfsClient().deleteBlobPath(path, leaseId, tracingContext);
+      getAbfsClient().incrementAbfsDeleteFile();
       operated = true;
     } finally {
       if (abfsLease != null) {

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/RenameAtomicity.java

@@ -267,6 +267,8 @@ public int preRename() throws AzureBlobFileSystemException {
         }
       }
       throw e;
+    } finally {
+      abfsClient.incrementAbfsCreateFile();
     }
   }
 
@@ -310,6 +312,8 @@ private void deleteRenamePendingJson() throws AzureBlobFileSystemException {
         return;
       }
       throw e;
+    } finally {
+      abfsClient.incrementAbfsDeleteFile();
     }
   }