Merge branch 'trunk' of https://github.com/anmolanmol1234/hadoop into HADOOP-19604

Author: anmolanmol1234

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -379,10 +379,6 @@ private static class DeprecatedKeyInfo {
       this.customMessage = customMessage;
     }
 
-    private final String getWarningMessage(String key) {
-      return getWarningMessage(key, null);
-    }
-
     /**
      * Method to provide the warning message. It gives the custom message if
      * non-null, and default message otherwise.
@@ -412,12 +408,9 @@ private String getWarningMessage(String key, String source) {
       return warningMessage;
     }
 
-    boolean getAndSetAccessed() {
-      return accessed.getAndSet(true);
-    }
-
-    public void clearAccessed() {
-      accessed.set(false);
+    void logDeprecation(String name, String source) {
+      LOG_DEPRECATION.info(getWarningMessage(name, source));
+      this.accessed.set(true);
     }
   }
 
@@ -728,12 +721,10 @@ private String[] handleDeprecation(DeprecationContext deprecations,
     }
     // Initialize the return value with requested name
     String[] names = new String[]{name};
-    // Deprecated keys are logged once and an updated names are returned
+    // Deprecated keys are logged and updated names are returned
     DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
     if (keyInfo != null) {
-      if (!keyInfo.getAndSetAccessed()) {
-        logDeprecation(keyInfo.getWarningMessage(name));
-      }
+      keyInfo.logDeprecation(name, null);
       // Override return value for deprecated keys
       names = keyInfo.newKeys;
     }
@@ -1462,13 +1453,6 @@ void logDeprecation(String message) {
     LOG_DEPRECATION.info(message);
   }
 
-  void logDeprecationOnce(String name, String source) {
-    DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
-    if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
-      LOG_DEPRECATION.info(keyInfo.getWarningMessage(name, source));
-    }
-  }
-
   /**
    * Unset a previously set property.
    * @param name the property name
@@ -2448,7 +2432,10 @@ private CredentialEntry getCredentialEntry(CredentialProvider provider,
     if (oldName != null) {
       entry = provider.getCredentialEntry(oldName);
       if (entry != null) {
-        logDeprecationOnce(oldName, provider.toString());
+        DeprecatedKeyInfo ki = getDeprecatedKeyInfo(oldName);
+        if (ki != null) {
+          ki.logDeprecation(oldName, provider.toString());
+        }
         return entry;
       }
     }
@@ -2459,7 +2446,7 @@ private CredentialEntry getCredentialEntry(CredentialProvider provider,
       for (String newName : keyInfo.newKeys) {
         entry = provider.getCredentialEntry(newName);
         if (entry != null) {
-          logDeprecationOnce(name, null);
+          keyInfo.logDeprecation(name, null);
           return entry;
         }
       }
@@ -3433,8 +3420,7 @@ void handleEndProperty() {
           deprecations.getDeprecatedKeyMap().get(confName);
 
       if (keyInfo != null) {
-        logDeprecation(keyInfo.getWarningMessage(confName, wrapper.toString()));
-        keyInfo.clearAccessed();
+        keyInfo.logDeprecation(confName, wrapper.toString());
         for (String key : keyInfo.newKeys) {
           // update new keys with deprecated key's value
           results.add(new ParsedItem(

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -384,6 +384,36 @@ public void testDeprecatedPropertyInXMLFileGeneratesLogMessage(@TempDir java.nio
     assertTrue(hasDeprecationMessage);
   }
 
+  @Test
+  public void testDeprecatedPropertyLogsWarningOnEveryUse(){
+    String oldProp = "test.deprecation.old.conf.b";
+    String newProp = "test.deprecation.new.conf.b";
+    Configuration.addDeprecation(oldProp, newProp);
+
+    TestAppender appender = new TestAppender();
+    Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
+    deprecationLogger.addAppender(appender);
+
+    try {
+      conf.set(oldProp, "b1");
+      conf.get(oldProp);
+      conf.set(oldProp, "b2");
+      conf.get(oldProp);
+      // Using the new property should not log a warning
+      conf.set(newProp, "b3");
+      conf.get(newProp);
+      conf.set(newProp, "b4");
+      conf.get(newProp);
+    } finally {
+      deprecationLogger.removeAppender(appender);
+    }
+
+    Pattern deprecationMsgPattern = Pattern.compile(oldProp + " is deprecated");
+    long count = appender.log.stream().map(LoggingEvent::getRenderedMessage)
+            .filter(msg -> deprecationMsgPattern.matcher(msg).find()).count();
+    assertEquals(4, count, "Expected exactly four warnings for deprecated property usage");
+  }
+
   /**
    * A simple appender for white box testing.
    */

hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml

@@ -77,6 +77,31 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.platform</groupId>
+      <artifactId>junit-platform-launcher</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c

@@ -263,8 +263,8 @@ jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...)
         // Too bad...
         return getPendingExceptionAndClear(env);
     }
-    exc = constructNewObjectOfClass(env, &out, "RuntimeException",
-        "(java/lang/String;)V", jstr);
+    exc = constructNewObjectOfClass(env, &out, "java/lang/RuntimeException",
+        "(Ljava/lang/String;)V", jstr);
     (*env)->DeleteLocalRef(env, jstr);
     // Again, we'll either get an out of memory exception or the
     // RuntimeException we wanted.

hadoop-tools/hadoop-aws/pom.xml

@@ -499,11 +499,6 @@
       <artifactId>wildfly-openssl</artifactId>
       <scope>runtime</scope>
     </dependency>
-    <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-inline</artifactId>
@@ -618,10 +613,5 @@
       <artifactId>junit-platform-launcher</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.junit.vintage</groupId>
-      <artifactId>junit-vintage-engine</artifactId>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 </project>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java

@@ -684,9 +684,8 @@ public void initialize(URI name, Configuration originalConf)
       s3ExpressStore = isS3ExpressStore(bucket, endpoint);
 
       // should the delete also purge uploads?
-      // happens if explicitly enabled, or if the store is S3Express storage.
       dirOperationsPurgeUploads = conf.getBoolean(DIRECTORY_OPERATIONS_PURGE_UPLOADS,
-          s3ExpressStore);
+          DIRECTORY_OPERATIONS_PURGE_UPLOADS_DEFAULT);
 
       this.isMultipartUploadEnabled = conf.getBoolean(MULTIPART_UPLOADS_ENABLED,
           DEFAULT_MULTIPART_UPLOAD_ENABLED);

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md

@@ -1218,10 +1218,29 @@ java.io.FileNotFoundException: Completing multi-part upload on fork-5/test/multi
 This can happen when all outstanding uploads have been aborted, including the
 active ones.
 
-If the bucket has a lifecycle policy of deleting multipart uploads, make sure
-that the expiry time of the deletion is greater than that required for all open
-writes to complete the write,
-*and for all jobs using the S3A committers to commit their work.*
+When working with S3A committers and multipart uploads (MPUs), consider these important guidelines:
+
+1. **Bucket Lifecycle Policies:**
+   - If your bucket has a lifecycle policy for deleting multipart uploads
+   - Set the deletion expiry time long enough to:
+     - Complete all open write operations
+     - Allow S3A committers to finish their commit process
+
+2. **Directory Operations and MPUs:**
+   - Setting `fs.s3a.directory.operations.purge.uploads=true` will abort all pending MPUs before directory cleanup
+   - For jobs using S3A committers:
+     - Set `fs.s3a.directory.operations.purge.uploads=false` when directories need to be overwritten before job completion
+     - This prevents accidental abortion of active uploads during the commit phase
+
+
+### S3 Express Store directory object not getting deleted
+
+When working with S3 Express store buckets (unlike standard S3 buckets), follow these steps to purge a directory object:
+
+1. Set `fs.s3a.directory.operations.purge.uploads=true` if you need to delete a directory object that has pending multipart uploads (MPUs).
+
+2. This setting ensures that all pending MPUs are aborted before the directory object is deleted, which is a requirement specific to S3 Express store buckets.
+
 
 ### Application hangs after reading a number of files
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java

@@ -28,10 +28,8 @@
 
 import org.apache.hadoop.conf.Configuration;
 
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 
 
 /**
@@ -49,14 +47,11 @@ public abstract class AbstractS3AMockTest {
               .build())
           .build();
 
-  @Rule
-  public ExpectedException exception = ExpectedException.none();
-
   protected S3AFileSystem fs;
   protected S3Client s3;
   protected Configuration conf;
 
-  @Before
+  @BeforeEach
   public void setup() throws Exception {
     conf = createConfiguration();
     fs = new S3AFileSystem();
@@ -97,7 +92,7 @@ public S3Client getS3Client() {
     return s3;
   }
 
-  @After
+  @AfterEach
   public void teardown() throws Exception {
     if (fs != null) {
       fs.close();

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestBlockingThreadPoolExecutorService.java

@@ -22,10 +22,9 @@
 import org.apache.hadoop.util.SemaphoredDelegatingExecutor;
 import org.apache.hadoop.util.StopWatch;
 
-import org.junit.AfterClass;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,11 +34,12 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
 
 /**
  * Basic test for S3A's blocking executor service.
  */
+@Timeout(60)
 public class ITestBlockingThreadPoolExecutorService {
 
   private static final Logger LOG = LoggerFactory.getLogger(
@@ -56,10 +56,7 @@ public class ITestBlockingThreadPoolExecutorService {
 
   private static BlockingThreadPoolExecutorService tpe;
 
-  @Rule
-  public Timeout testTimeout = new Timeout(60, TimeUnit.SECONDS);
-
-  @AfterClass
+  @AfterAll
   public static void afterClass() throws Exception {
     ensureDestroyed();
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestEMRFSCompatibility.java

@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.s3a;
 
 import org.assertj.core.api.Assertions;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import org.apache.hadoop.fs.Path;