HBASE-26328 Clone snapshot doesn't load reference files into FILE SFT impl (#3749)

Signed-off-by: Duo Zhang <[email protected]>

Author: wchevreuil

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java

@@ -202,7 +202,6 @@ public static boolean isHFileLink(final Path path) {
     return isHFileLink(path.getName());
   }
 
-
   /**
    * @param fileName File name to check.
    * @return True if the path is a HFileLink.
@@ -323,10 +322,10 @@ public static String createHFileLinkName(final TableName tableName,
    * @param dstFamilyPath - Destination path (table/region/cf/)
    * @param hfileRegionInfo - Linked HFile Region Info
    * @param hfileName - Linked HFile name
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
    */
-  public static boolean create(final Configuration conf, final FileSystem fs,
+  public static String create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final RegionInfo hfileRegionInfo,
       final String hfileName) throws IOException {
     return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
@@ -344,10 +343,10 @@ public static boolean create(final Configuration conf, final FileSystem fs,
    * @param hfileRegionInfo - Linked HFile Region Info
    * @param hfileName - Linked HFile name
    * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
    */
-  public static boolean create(final Configuration conf, final FileSystem fs,
+  public static String create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final RegionInfo hfileRegionInfo,
       final String hfileName, final boolean createBackRef) throws IOException {
     TableName linkedTable = hfileRegionInfo.getTable();
@@ -367,17 +366,18 @@ public static boolean create(final Configuration conf, final FileSystem fs,
    * @param linkedTable - Linked Table Name
    * @param linkedRegion - Linked Region Name
    * @param hfileName - Linked HFile name
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
    */
-  public static boolean create(final Configuration conf, final FileSystem fs,
+  public static String create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
       final String hfileName) throws IOException {
     return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
   }
 
   /**
-   * Create a new HFileLink
+   * Create a new HFileLink. In the event of link creation failure, this method throws an
+   * IOException, so that the calling upper laying can decide on how to proceed with this.
    *
    * <p>It also adds a back-reference to the hfile back-reference directory
    * to simplify the reference-count and the cleaning process.
@@ -389,10 +389,10 @@ public static boolean create(final Configuration conf, final FileSystem fs,
    * @param linkedRegion - Linked Region Name
    * @param hfileName - Linked HFile name
    * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
    */
-  public static boolean create(final Configuration conf, final FileSystem fs,
+  public static String create(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
       final String hfileName, final boolean createBackRef) throws IOException {
     String familyName = dstFamilyPath.getName();
@@ -420,7 +420,9 @@ public static boolean create(final Configuration conf, final FileSystem fs,
     }
     try {
       // Create the link
-      return fs.createNewFile(new Path(dstFamilyPath, name));
+      if(fs.createNewFile(new Path(dstFamilyPath, name))){
+        return name;
+      }
     } catch (IOException e) {
       LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
       // Revert the reference if the link creation failed
@@ -429,25 +431,8 @@ public static boolean create(final Configuration conf, final FileSystem fs,
       }
       throw e;
     }
-  }
-
-  /**
-   * Create a new HFileLink starting from a hfileLink name
-   *
-   * <p>It also adds a back-reference to the hfile back-reference directory
-   * to simplify the reference-count and the cleaning process.
-   *
-   * @param conf {@link Configuration} to read for the archive directory name
-   * @param fs {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
-   */
-  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
-      final Path dstFamilyPath, final String hfileLinkName)
-          throws IOException {
-    return createFromHFileLink(conf, fs, dstFamilyPath, hfileLinkName, true);
+    throw new IOException("File link=" + name + " already exists under " +
+      dstFamilyPath + " folder.");
   }
 
   /**
@@ -461,10 +446,10 @@ public static boolean createFromHFileLink(final Configuration conf, final FileSy
    * @param dstFamilyPath - Destination path (table/region/cf/)
    * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
    * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return true if the file is created, otherwise the file exists.
-   * @throws IOException on file or parent directory creation failure
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
    */
-  public static boolean createFromHFileLink(final Configuration conf, final FileSystem fs,
+  public static String createFromHFileLink(final Configuration conf, final FileSystem fs,
       final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
           throws IOException {
     Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);

hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java

@@ -40,7 +40,6 @@
 import org.apache.hadoop.hbase.master.RegionState;
 import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
 import org.apache.hadoop.hbase.master.procedure.CreateTableProcedure.CreateHdfsRegions;
-import org.apache.hadoop.hbase.mob.MobUtils;
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
 import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
@@ -453,56 +452,25 @@ private List<RegionInfo> createFsLayout(
     List<RegionInfo> newRegions,
     final CreateHdfsRegions hdfsRegionHandler) throws IOException {
     final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
-    final Path tempdir = mfs.getTempDir();
 
     // 1. Create Table Descriptor
     // using a copy of descriptor, table will be created enabling first
-    final Path tempTableDir = CommonFSUtils.getTableDir(tempdir, tableDescriptor.getTableName());
-    if (CommonFSUtils.isExists(mfs.getFileSystem(), tempTableDir)) {
+    final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(),
+      tableDescriptor.getTableName());
+    if (CommonFSUtils.isExists(mfs.getFileSystem(), tableDir)) {
       // if the region dirs exist, will cause exception and unlimited retry (see HBASE-24546)
-      LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tempTableDir);
-      CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tempTableDir);
+      LOG.warn("temp table dir already exists on disk: {}, will be deleted.", tableDir);
+      CommonFSUtils.deleteDirectory(mfs.getFileSystem(), tableDir);
     }
-    ((FSTableDescriptors) (env.getMasterServices().getTableDescriptors()))
-      .createTableDescriptorForTableDirectory(tempTableDir,
-        TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false);
+    ((FSTableDescriptors)(env.getMasterServices().getTableDescriptors()))
+      .createTableDescriptorForTableDirectory(tableDir,
+              TableDescriptorBuilder.newBuilder(tableDescriptor).build(), false);
 
     // 2. Create Regions
     newRegions = hdfsRegionHandler.createHdfsRegions(
-      env, tempdir, tableDescriptor.getTableName(), newRegions);
-
-    // 3. Move Table temp directory to the hbase root location
-    CreateTableProcedure.moveTempDirectoryToHBaseRoot(env, tableDescriptor, tempTableDir);
-    // Move Table temp mob directory to the hbase root location
-    Path tempMobTableDir = MobUtils.getMobTableDir(tempdir, tableDescriptor.getTableName());
-    if (mfs.getFileSystem().exists(tempMobTableDir)) {
-      moveTempMobDirectoryToHBaseRoot(mfs, tableDescriptor, tempMobTableDir);
-    }
-    return newRegions;
-  }
+      env, mfs.getRootDir(), tableDescriptor.getTableName(), newRegions);
 
-  /**
-   * Move table temp mob directory to the hbase root location
-   * @param mfs The master file system
-   * @param tableDescriptor The table to operate on
-   * @param tempMobTableDir The temp mob directory of table
-   * @throws IOException If failed to move temp mob dir to hbase root dir
-   */
-  private void moveTempMobDirectoryToHBaseRoot(final MasterFileSystem mfs,
-      final TableDescriptor tableDescriptor, final Path tempMobTableDir) throws IOException {
-    FileSystem fs = mfs.getFileSystem();
-    final Path tableMobDir =
-        MobUtils.getMobTableDir(mfs.getRootDir(), tableDescriptor.getTableName());
-    if (!fs.delete(tableMobDir, true) && fs.exists(tableMobDir)) {
-      throw new IOException("Couldn't delete mob table " + tableMobDir);
-    }
-    if (!fs.exists(tableMobDir.getParent())) {
-      fs.mkdirs(tableMobDir.getParent());
-    }
-    if (!fs.rename(tempMobTableDir, tableMobDir)) {
-      throw new IOException("Unable to move mob table from temp=" + tempMobTableDir
-          + " to hbase root=" + tableMobDir);
-    }
+    return newRegions;
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java

@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.Collection;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;