HADOOP-13230 operation cost tests

* defined constants for the Head/List cost of various ops,
use them in the asserts over "1", "2" etc to make clearer origin
of operations

* Cut the standalone marker HEAD request. We don't need it, ever.

That will simplify backporting too; I have a clear plan of the minimum
needed for backporting to branch-3.3 and then again for 3.0-3.2

Change-Id: I33f9ccf2b477c027afe70137993c8344554f07c6

Author: steveloughran

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -2544,7 +2544,7 @@ private void createFakeDirectoryIfNecessary(Path f)
     // we only make the LIST call; the codepaths to get here should not
     // be reached if there is an empty dir marker -and if they do, it
     // is mostly harmless to create a new one.
-    if (!key.isEmpty() && !s3Exists(f, EnumSet.of(StatusProbeEnum.List))) {
+    if (!key.isEmpty() && !s3Exists(f, StatusProbeEnum.DIRECTORIES)) {
       LOG.debug("Creating new fake directory at {}", f);
       createFakeDirectory(key);
     }
@@ -3058,8 +3058,7 @@ S3AFileStatus s3GetFileStatus(final Path path,
     }
 
     // execute the list
-    boolean executeList = probes.contains(StatusProbeEnum.List);
-    if (executeList) {
+    if (probes.contains(StatusProbeEnum.List)) {
       try {
         // this will find a marker dir / as well as an entry.
         // When making a simple "is this a dir check" all is good.
@@ -3119,45 +3118,6 @@ S3AFileStatus s3GetFileStatus(final Path path,
       }
     }
 
-    // Neither any normal file HEAD nor a LIST found an object
-    // Look for the dir marker
-    // TODO: decide whether to cut or not. We currently skip this entire probe.
-    boolean checkMarker = false && probes.contains(StatusProbeEnum.DirMarker);
-    if (!key.isEmpty() && checkMarker) {
-      String newKey = maybeAddTrailingSlash(key);
-      try {
-        ObjectMetadata meta = getObjectMetadata(newKey);
-
-        if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
-          LOG.debug("Found file (with /): fake directory");
-          // this used to self-declare as empty; now it decides
-          // based on whether a list was also executed in the current
-          // series of probes
-          return new S3AFileStatus(
-              executeList
-                  ? Tristate.TRUE
-                  : Tristate.UNKNOWN,
-              path, username);
-        } else {
-          LOG.warn("Found file (with /): real file? should not happen: {}",
-              key);
-
-          return new S3AFileStatus(meta.getContentLength(),
-              dateToLong(meta.getLastModified()),
-              path,
-              getDefaultBlockSize(path),
-              username,
-              meta.getETag(),
-              meta.getVersionId());
-        }
-      } catch (AmazonServiceException e) {
-        if (e.getStatusCode() != SC_404 || isUnknownBucket(e)) {
-          throw translateException("getFileStatus", newKey, e);
-        }
-      } catch (AmazonClientException e) {
-        throw translateException("getFileStatus", newKey, e);
-      }
-    }
     LOG.debug("Not Found: {}", path);
     throw new FileNotFoundException("No such file or directory: " + path);
   }
@@ -3560,6 +3520,7 @@ private CopyResult copyFile(String srcKey, String dstKey, long size,
           copyObjectRequest.setNewObjectMetadata(dstom);
           Optional.ofNullable(srcom.getStorageClass())
               .ifPresent(copyObjectRequest::setStorageClass);
+          incrementStatistic(OBJECT_COPY_REQUESTS);
           Copy copy = transfers.copy(copyObjectRequest);
           copy.addProgressListener(progressListener);
           CopyOutcome copyOutcome = CopyOutcome.waitForCopy(copy);

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StatusProbeEnum.java

@@ -34,32 +34,24 @@ public enum StatusProbeEnum {
   List;
 
   /** Look for files and directories. */
-  public static final Set<StatusProbeEnum> FILES_AND_DIRECTORIES =
+  public static final Set<StatusProbeEnum> ALL =
       EnumSet.of(Head, List);
 
-  /**
-   * This used to mean "all probes", now it means "all file type.
-   */
-  public static final Set<StatusProbeEnum> ALL = FILES_AND_DIRECTORIES;
-
-  /** Skip the HEAD and only look for directories. */
-  public static final Set<StatusProbeEnum> DIRECTORIES =
-      EnumSet.of(DirMarker, List);
-
-  /** We only want the HEAD or dir marker. */
-  public static final Set<StatusProbeEnum> HEAD_OR_DIR_MARKER =
-      EnumSet.of(Head, DirMarker);
-
   /** We only want the HEAD. */
   public static final Set<StatusProbeEnum> HEAD_ONLY =
       EnumSet.of(Head);
 
-  /** We only want the dir marker. */
-  public static final Set<StatusProbeEnum> DIR_MARKER_ONLY =
-      EnumSet.of(DirMarker);
-
-  /** We only want the dir marker. */
+  /** List operation only. */
   public static final Set<StatusProbeEnum> LIST_ONLY =
       EnumSet.of(List);
 
+  /** Look for files and directories. */
+  public static final Set<StatusProbeEnum> FILE =
+      HEAD_ONLY;
+
+  /** Skip the HEAD and only look for directories. */
+  public static final Set<StatusProbeEnum> DIRECTORIES =
+      LIST_ONLY;
+
+
 }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEmptyDirectory.java

@@ -80,7 +80,7 @@ public void testDirectoryBecomesNonEmpty() throws Exception {
   private S3AFileStatus getS3AFileStatus(S3AFileSystem fs, Path p) throws
       IOException {
     return fs.innerGetFileStatus(p, true,
-        StatusProbeEnum.FILES_AND_DIRECTORIES);
+        StatusProbeEnum.ALL);
   }
 
 }