HBASE-28616 Remove/Deprecated the rs.* related configuration in TableOutputFormat (#5946)

Apache9 · Apache9 · commit 771e760ed8da · 2024-05-29T15:25:23.000+08:00
Signed-off-by: Andrew Purtell <apurtell@apache.org> Signed-off-by: Pankaj Kumar <pankajkumar@apache.org> Reviewed-by: Subrat Mishra <subrat.mishra@gmail.com> (cherry picked from commit 771e1d9)
diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithVisibility.java
@@ -273,7 +273,7 @@ public int runCopier(String outputDir) throws Exception {
       }
       job.getConfiguration().setBoolean("mapreduce.map.speculative", false);
       job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false);
-      TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null);
+      TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job);
       TableMapReduceUtil.addDependencyJars(job);
       TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
       TableMapReduceUtil.initCredentials(job);
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
@@ -180,8 +180,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
       }
     } else {
       initCopyTableMapperReducerJob(job, scan);
-      TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null,
-        null);
+      TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress);
     }
 
     return job;
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -167,8 +167,7 @@ public Job createSubmittableJob(String[] args) throws IOException {
     } else {
       // No reducers. Just write straight to table. Call initTableReducerJob
       // because it sets up the TableOutputFormat.
-      TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster,
-        null, null);
+      TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster);
 
       // would be nice to add an option for bulk load instead
     }
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -569,7 +569,7 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
    */
   public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
     Job job, Class partitioner) throws IOException {
-    initTableReducerJob(table, reducer, job, partitioner, null, null, null);
+    initTableReducerJob(table, reducer, job, partitioner, null);
   }
 
   /**
@@ -589,15 +589,11 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
    *                      <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
    *             hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
    * </code>           such as <code>server,server2,server3:2181:/hbase</code>.
-   * @param serverClass   redefined hbase.regionserver.class
-   * @param serverImpl    redefined hbase.regionserver.impl
    * @throws IOException When determining the region count fails.
    */
   public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
-    Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
-    throws IOException {
-    initTableReducerJob(table, reducer, job, partitioner, quorumAddress, serverClass, serverImpl,
-      true);
+    Job job, Class partitioner, String quorumAddress) throws IOException {
+    initTableReducerJob(table, reducer, job, partitioner, quorumAddress, true);
   }
 
   /**
@@ -617,16 +613,13 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
    *                          <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
    *             hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
    * </code>               such as <code>server,server2,server3:2181:/hbase</code>.
-   * @param serverClass       redefined hbase.regionserver.class
-   * @param serverImpl        redefined hbase.regionserver.impl
    * @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
    *                          the distributed cache (tmpjars).
    * @throws IOException When determining the region count fails.
    */
   public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
-    Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
-    boolean addDependencyJars) throws IOException {
-
+    Job job, Class partitioner, String quorumAddress, boolean addDependencyJars)
+    throws IOException {
     Configuration conf = job.getConfiguration();
     HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));
     job.setOutputFormatClass(TableOutputFormat.class);
@@ -640,10 +633,6 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
       ZKConfig.validateClusterKey(quorumAddress);
       conf.set(TableOutputFormat.QUORUM_ADDRESS, quorumAddress);
     }
-    if (serverClass != null && serverImpl != null) {
-      conf.set(TableOutputFormat.REGION_SERVER_CLASS, serverClass);
-      conf.set(TableOutputFormat.REGION_SERVER_IMPL, serverImpl);
-    }
     job.setOutputKeyClass(ImmutableBytesWritable.class);
     job.setOutputValueClass(Writable.class);
     if (partitioner == HRegionPartitioner.class) {
@@ -663,6 +652,72 @@ public static void initTableReducerJob(String table, Class<? extends TableReduce
     initCredentials(job);
   }
 
+  /**
+   * Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
+   * @param table         The output table.
+   * @param reducer       The reducer class to use.
+   * @param job           The current job to adjust. Make sure the passed job is carrying all
+   *                      necessary HBase configuration.
+   * @param partitioner   Partitioner to use. Pass <code>null</code> to use default partitioner.
+   * @param quorumAddress Distant cluster to write to; default is null for output to the cluster
+   *                      that is designated in <code>hbase-site.xml</code>. Set this String to the
+   *                      zookeeper ensemble of an alternate remote cluster when you would have the
+   *                      reduce write a cluster that is other than the default; e.g. copying tables
+   *                      between clusters, the source would be designated by
+   *                      <code>hbase-site.xml</code> and this param would have the ensemble address
+   *                      of the remote cluster. The format to pass is particular. Pass
+   *                      <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
+   *             hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
+   * </code>           such as <code>server,server2,server3:2181:/hbase</code>.
+   * @param serverClass   redefined hbase.regionserver.class
+   * @param serverImpl    redefined hbase.regionserver.impl
+   * @throws IOException When determining the region count fails.
+   * @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
+   *             {@code serverImpl} do not take effect any more, just use
+   *             {@link #initTableReducerJob(String, Class, Job, Class, String)} instead.
+   * @see #initTableReducerJob(String, Class, Job, Class, String)
+   */
+  @Deprecated
+  public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
+    Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl)
+    throws IOException {
+    initTableReducerJob(table, reducer, job, partitioner, quorumAddress);
+  }
+
+  /**
+   * Use this before submitting a TableReduce job. It will appropriately set up the JobConf.
+   * @param table             The output table.
+   * @param reducer           The reducer class to use.
+   * @param job               The current job to adjust. Make sure the passed job is carrying all
+   *                          necessary HBase configuration.
+   * @param partitioner       Partitioner to use. Pass <code>null</code> to use default partitioner.
+   * @param quorumAddress     Distant cluster to write to; default is null for output to the cluster
+   *                          that is designated in <code>hbase-site.xml</code>. Set this String to
+   *                          the zookeeper ensemble of an alternate remote cluster when you would
+   *                          have the reduce write a cluster that is other than the default; e.g.
+   *                          copying tables between clusters, the source would be designated by
+   *                          <code>hbase-site.xml</code> and this param would have the ensemble
+   *                          address of the remote cluster. The format to pass is particular. Pass
+   *                          <code> &lt;hbase.zookeeper.quorum&gt;:&lt;
+   *             hbase.zookeeper.client.port&gt;:&lt;zookeeper.znode.parent&gt;
+   * </code>               such as <code>server,server2,server3:2181:/hbase</code>.
+   * @param serverClass       redefined hbase.regionserver.class
+   * @param serverImpl        redefined hbase.regionserver.impl
+   * @param addDependencyJars upload HBase jars and jars for any of the configured job classes via
+   *                          the distributed cache (tmpjars).
+   * @throws IOException When determining the region count fails.
+   * @deprecated Since 2.5.9, 2.6.1, 2.7.0, will be removed in 4.0.0. The {@code serverClass} and
+   *             {@code serverImpl} do not take effect any more, just use
+   *             {@link #initTableReducerJob(String, Class, Job, Class, String, boolean)} instead.
+   * @see #initTableReducerJob(String, Class, Job, Class, String, boolean)
+   */
+  @Deprecated
+  public static void initTableReducerJob(String table, Class<? extends TableReducer> reducer,
+    Job job, Class partitioner, String quorumAddress, String serverClass, String serverImpl,
+    boolean addDependencyJars) throws IOException {
+    initTableReducerJob(table, reducer, job, partitioner, quorumAddress, addDependencyJars);
+  }
+
   /**
    * Ensures that the given number of reduce tasks for the given job configuration does not exceed
    * the number of regions for the given table.
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -66,16 +66,26 @@ public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation> implemen
    * Optional job parameter to specify a peer cluster. Used specifying remote cluster when copying
    * between hbase clusters (the source is picked up from <code>hbase-site.xml</code>).
    * @see TableMapReduceUtil#initTableReducerJob(String, Class, org.apache.hadoop.mapreduce.Job,
-   *      Class, String, String, String)
+   *      Class, String)
    */
   public static final String QUORUM_ADDRESS = OUTPUT_CONF_PREFIX + "quorum";
 
   /** Optional job parameter to specify peer cluster's ZK client port */
   public static final String QUORUM_PORT = OUTPUT_CONF_PREFIX + "quorum.port";
 
-  /** Optional specification of the rs class name of the peer cluster */
+  /**
+   * Optional specification of the rs class name of the peer cluster.
+   * @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
+   *             long ago, see HBASE-6044.
+   */
+  @Deprecated
   public static final String REGION_SERVER_CLASS = OUTPUT_CONF_PREFIX + "rs.class";
-  /** Optional specification of the rs impl name of the peer cluster */
+  /**
+   * Optional specification of the rs impl name of the peer cluster
+   * @deprecated Since 2.5.9, 2.6.1 and 2.7.0, will be removed in 4.0.0. Does not take effect from
+   *             long ago, see HBASE-6044.
+   */
+  @Deprecated
   public static final String REGION_SERVER_IMPL = OUTPUT_CONF_PREFIX + "rs.impl";
 
   /** The configuration. */
@@ -208,15 +218,9 @@ public void setConf(Configuration otherConf) {
 
     String address = otherConf.get(QUORUM_ADDRESS);
     int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
-    String serverClass = otherConf.get(REGION_SERVER_CLASS);
-    String serverImpl = otherConf.get(REGION_SERVER_IMPL);
 
     try {
       this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);
-
-      if (serverClass != null) {
-        this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
-      }
       if (zkClientPort != 0) {
         this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
       }

Original file line number	Diff line number	Diff line change
`@@ -273,7 +273,7 @@ public int runCopier(String outputDir) throws Exception {`
`273`	`273`	`}`
`274`	`274`	`job.getConfiguration().setBoolean("mapreduce.map.speculative", false);`
`275`	`275`	`job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false);`
`276`		`- TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null);`
	`276`	`+ TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job);`
`277`	`277`	`TableMapReduceUtil.addDependencyJars(job);`
`278`	`278`	`TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);`
`279`	`279`	`TableMapReduceUtil.initCredentials(job);`
Original file line number	Diff line number	Diff line change
`@@ -180,8 +180,7 @@ public Job createSubmittableJob(String[] args) throws IOException {`
`180`	`180`	`}`
`181`	`181`	`} else {`
`182`	`182`	`initCopyTableMapperReducerJob(job, scan);`
`183`		`- TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress, null,`
`184`		`- null);`
	`183`	`+ TableMapReduceUtil.initTableReducerJob(dstTableName, null, job, null, peerAddress);`
`185`	`184`	`}`
`186`	`185`
`187`	`186`	`return job;`
Original file line number	Diff line number	Diff line change
`@@ -167,8 +167,7 @@ public Job createSubmittableJob(String[] args) throws IOException {`
`167`	`167`	`} else {`
`168`	`168`	`// No reducers. Just write straight to table. Call initTableReducerJob`
`169`	`169`	`// because it sets up the TableOutputFormat.`
`170`		`- TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster,`
`171`		`- null, null);`
	`170`	`+ TableMapReduceUtil.initTableReducerJob(targetTableName, null, job, null, targetZkCluster);`
`172`	`171`
`173`	`172`	`// would be nice to add an option for bulk load instead`
`174`	`173`	`}`