Skip to content

Commit 10cb001

Browse files
author
Prathyusha Garre
committed
Fix uts to consume sft as part of Store file constructors
1 parent c41b878 commit 10cb001

37 files changed

+496
-280
lines changed

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ public String getStoragePolicyName(String familyName) {
254254

255255
return null;
256256
}
257-
257+
258258
/**
259259
* Returns the store files' LocatedFileStatus which available for the family. This methods
260260
* performs the filtering based on the valid store files.
@@ -311,7 +311,7 @@ StoreFileInfo getStoreFileInfo(final String familyName, final String fileName,
311311
return ServerRegionReplicaUtil.getStoreFileInfo(conf, fs, regionInfo, regionInfoForFs,
312312
familyName, new Path(familyDir, fileName), tracker);
313313
}
314-
314+
315315
/**
316316
* Returns true if the specified family has reference files
317317
* @param familyName Column Family Name
@@ -698,7 +698,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte
698698
}
699699
// must create back reference here
700700
HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
701-
hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
701+
hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
702702
Path path =
703703
new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
704704
LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java‎

Lines changed: 43 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.hadoop.hbase.io.Reference;
3636
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
3737
import org.apache.hadoop.hbase.io.hfile.HFileInfo;
38+
import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
3839
import org.apache.hadoop.hbase.io.hfile.ReaderContext;
3940
import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType;
4041
import org.apache.hadoop.hbase.io.hfile.ReaderContextBuilder;
@@ -124,6 +125,33 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path i
124125
this(conf, fs, null, initialPath, primaryReplica, sft);
125126
}
126127

128+
public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath,
129+
final boolean primaryReplica, boolean isHfile) throws IOException {
130+
if (HFileLink.isHFileLink(initialPath) || isReference(initialPath)) {
131+
throw new InvalidHFileException("Path " + initialPath + " is a Hfile link or a Regerence");
132+
}
133+
assert fs != null;
134+
assert initialPath != null;
135+
assert conf != null;
136+
137+
this.fs = fs;
138+
this.conf = conf;
139+
this.initialPath = fs.makeQualified(initialPath);
140+
this.primaryReplica = primaryReplica;
141+
this.noReadahead =
142+
this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD);
143+
Path p = initialPath;
144+
if (isHFile(p) || isMobFile(p) || isMobRefFile(p)) {
145+
FileStatus fStatus = fs.getFileStatus(initialPath);
146+
this.createdTimestamp = fStatus.getModificationTime();
147+
this.size = fStatus.getLen();
148+
this.reference = null;
149+
this.link = null;
150+
} else {
151+
throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
152+
}
153+
}
154+
127155
private StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
128156
final Path initialPath, final boolean primaryReplica, final StoreFileTracker sft)
129157
throws IOException {
@@ -214,7 +242,7 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt
214242
this.noReadahead =
215243
this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD);
216244
}
217-
245+
218246
/**
219247
* Create a Store File Info from an HFileLink and a Reference
220248
* @param conf The {@link Configuration} to use
@@ -223,19 +251,20 @@ public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileSt
223251
* @param reference The reference instance
224252
* @param link The link instance
225253
*/
226-
public StoreFileInfo(final Configuration conf, final FileSystem fs, final long createdTimestamp,
227-
final Path initialPath, final long size, final Reference reference, final HFileLink link,
228-
final boolean primaryReplica) {
229-
this.fs = fs;
230-
this.conf = conf;
231-
this.primaryReplica = primaryReplica;
232-
this.initialPath = initialPath;
233-
this.createdTimestamp = createdTimestamp;
234-
this.size = size;
235-
this.reference = reference;
236-
this.link = link;
237-
this.noReadahead = this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD);
238-
}
254+
public StoreFileInfo(final Configuration conf, final FileSystem fs, final long createdTimestamp,
255+
final Path initialPath, final long size, final Reference reference, final HFileLink link,
256+
final boolean primaryReplica) {
257+
this.fs = fs;
258+
this.conf = conf;
259+
this.primaryReplica = primaryReplica;
260+
this.initialPath = initialPath;
261+
this.createdTimestamp = createdTimestamp;
262+
this.size = size;
263+
this.reference = reference;
264+
this.link = link;
265+
this.noReadahead =
266+
this.conf.getBoolean(STORE_FILE_READER_NO_READAHEAD, DEFAULT_STORE_FILE_READER_NO_READAHEAD);
267+
}
239268

240269
@Override
241270
public Configuration getConf() {

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/DefaultStoreFileTracker.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,10 @@
2222
import java.util.Collection;
2323
import java.util.Collections;
2424
import java.util.List;
25-
2625
import org.apache.hadoop.conf.Configuration;
2726
import org.apache.hadoop.fs.FileStatus;
28-
import org.apache.hadoop.fs.FileSystem;
2927
import org.apache.hadoop.fs.Path;
3028
import org.apache.hadoop.hbase.HConstants;
31-
import org.apache.hadoop.hbase.io.HFileLink;
32-
import org.apache.hadoop.hbase.io.Reference;
3329
import org.apache.hadoop.hbase.regionserver.StoreContext;
3430
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
3531
import org.apache.hadoop.hbase.util.CommonFSUtils;

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import java.io.IOException;
2121
import java.util.Collection;
2222
import java.util.List;
23-
2423
import org.apache.hadoop.fs.FileStatus;
2524
import org.apache.hadoop.fs.Path;
2625
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
@@ -102,9 +101,13 @@ void replace(Collection<StoreFileInfo> compactedFiles, Collection<StoreFileInfo>
102101
Reference createReference(Reference reference, Path path) throws IOException;
103102

104103
Reference readReference(Path path) throws IOException;
105-
104+
106105
boolean hasReferences(final String familyName) throws IOException;
107-
108-
StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath, final boolean primaryReplica) throws IOException;
109-
106+
107+
StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPath,
108+
final boolean primaryReplica) throws IOException;
109+
110+
StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica)
111+
throws IOException;
112+
110113
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java‎

Lines changed: 53 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
import java.io.InputStream;
2626
import java.util.Collection;
2727
import java.util.List;
28-
2928
import org.apache.commons.io.IOUtils;
3029
import org.apache.hadoop.conf.Configuration;
3130
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -47,13 +46,14 @@
4746
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
4847
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
4948
import org.apache.hadoop.hbase.regionserver.StoreUtils;
50-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
5149
import org.apache.hadoop.hbase.util.CommonFSUtils;
5250
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
5351
import org.apache.yetus.audience.InterfaceAudience;
5452
import org.slf4j.Logger;
5553
import org.slf4j.LoggerFactory;
5654

55+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
56+
5757
/**
5858
* Base class for all store file tracker.
5959
* <p/>
@@ -264,48 +264,57 @@ public Reference readReference(final Path p) throws IOException {
264264
}
265265

266266
@Override
267-
public StoreFileInfo getStoreFileInfo(FileStatus fileStatus, Path initialPath, boolean primaryReplica)
268-
throws IOException {
269-
FileSystem fs = this.ctx.getRegionFileSystem().getFileSystem();
270-
assert fs != null;
271-
assert initialPath != null;
272-
assert conf != null;
273-
Reference reference = null;
274-
HFileLink link = null;
275-
long createdTimestamp = 0;
276-
long size = 0;
277-
Path p = initialPath;
278-
if (HFileLink.isHFileLink(p)) {
279-
// HFileLink
280-
reference = null;
281-
link = HFileLink.buildFromHFileLinkPattern(conf, p);
282-
LOG.trace("{} is a link", p);
283-
} else if (StoreFileInfo.isReference(p)) {
284-
reference = readReference(p);
285-
Path referencePath = StoreFileInfo.getReferredToFile(p);
286-
if (HFileLink.isHFileLink(referencePath)) {
287-
// HFileLink Reference
288-
link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
289-
} else {
290-
// Reference
291-
link = null;
292-
}
293-
LOG.trace("{} is a {} reference to {}", p, reference.getFileRegion(), referencePath);
294-
} else if (StoreFileInfo.isHFile(p) || StoreFileInfo.isMobFile(p) || StoreFileInfo.isMobRefFile(p)) {
295-
// HFile
296-
if (fileStatus != null) {
297-
createdTimestamp = fileStatus.getModificationTime();
298-
size = fileStatus.getLen();
299-
} else {
300-
FileStatus fStatus = fs.getFileStatus(initialPath);
301-
createdTimestamp = fStatus.getModificationTime();
302-
size = fStatus.getLen();
303-
}
304-
} else {
305-
throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
306-
}
307-
return new StoreFileInfo(conf, fs, createdTimestamp, initialPath, size, reference, link, isPrimaryReplica);
308-
}
267+
public StoreFileInfo getStoreFileInfo(Path initialPath, boolean primaryReplica)
268+
throws IOException {
269+
return getStoreFileInfo(null, initialPath, primaryReplica);
270+
}
271+
272+
@Override
273+
public StoreFileInfo getStoreFileInfo(FileStatus fileStatus, Path initialPath,
274+
boolean primaryReplica) throws IOException {
275+
FileSystem fs = this.ctx.getRegionFileSystem().getFileSystem();
276+
assert fs != null;
277+
assert initialPath != null;
278+
assert conf != null;
279+
Reference reference = null;
280+
HFileLink link = null;
281+
long createdTimestamp = 0;
282+
long size = 0;
283+
Path p = initialPath;
284+
if (HFileLink.isHFileLink(p)) {
285+
// HFileLink
286+
reference = null;
287+
link = HFileLink.buildFromHFileLinkPattern(conf, p);
288+
LOG.trace("{} is a link", p);
289+
} else if (StoreFileInfo.isReference(p)) {
290+
reference = readReference(p);
291+
Path referencePath = StoreFileInfo.getReferredToFile(p);
292+
if (HFileLink.isHFileLink(referencePath)) {
293+
// HFileLink Reference
294+
link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
295+
} else {
296+
// Reference
297+
link = null;
298+
}
299+
LOG.trace("{} is a {} reference to {}", p, reference.getFileRegion(), referencePath);
300+
} else
301+
if (StoreFileInfo.isHFile(p) || StoreFileInfo.isMobFile(p) || StoreFileInfo.isMobRefFile(p)) {
302+
// HFile
303+
if (fileStatus != null) {
304+
createdTimestamp = fileStatus.getModificationTime();
305+
size = fileStatus.getLen();
306+
} else {
307+
FileStatus fStatus = fs.getFileStatus(initialPath);
308+
createdTimestamp = fStatus.getModificationTime();
309+
size = fStatus.getLen();
310+
}
311+
} else {
312+
throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
313+
}
314+
return new StoreFileInfo(conf, fs, createdTimestamp, initialPath, size, reference, link,
315+
isPrimaryReplica);
316+
}
317+
309318
/**
310319
* For primary replica, we will call load once when opening a region, and the implementation could
311320
* choose to do some cleanup work. So here we use {@code readOnly} to indicate that whether you

‎hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import java.util.Set;
3333
import java.util.TreeMap;
3434
import java.util.concurrent.ThreadPoolExecutor;
35-
3635
import org.apache.hadoop.conf.Configuration;
3736
import org.apache.hadoop.fs.FileStatus;
3837
import org.apache.hadoop.fs.FileSystem;
@@ -62,21 +61,23 @@
6261
import org.apache.hadoop.hbase.security.access.Permission;
6362
import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
6463
import org.apache.hadoop.hbase.security.access.TablePermission;
65-
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
66-
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
67-
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
6864
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper.RestoreMetaChanges;
6965
import org.apache.hadoop.hbase.util.Bytes;
7066
import org.apache.hadoop.hbase.util.CommonFSUtils;
7167
import org.apache.hadoop.hbase.util.FSUtils;
7268
import org.apache.hadoop.hbase.util.ModifyRegionUtils;
7369
import org.apache.hadoop.hbase.util.Pair;
7470
import org.apache.hadoop.io.IOUtils;
75-
import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
7671
import org.apache.yetus.audience.InterfaceAudience;
7772
import org.slf4j.Logger;
7873
import org.slf4j.LoggerFactory;
7974

75+
import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap;
76+
77+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
78+
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
79+
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
80+
8081
/**
8182
* Helper to Restore/Clone a Snapshot
8283
* <p>

‎hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -616,4 +616,4 @@ static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) {
616616
private static int getSnapshotFormat(final SnapshotDescription desc) {
617617
return desc.hasVersion() ? desc.getVersion() : SnapshotManifestV1.DESCRIPTOR_VERSION;
618618
}
619-
}
619+
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifestV1.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,4 +216,4 @@ static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf, fi
216216
}
217217
return manifest.build();
218218
}
219-
}
219+
}

‎hbase-server/src/main/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,4 +174,4 @@ HRegionFileSystem getFileSystem() throws IOException {
174174
public String toString() {
175175
return "region: " + region.getEncodedName() + " store(s): " + stores;
176176
}
177-
}
177+
}

‎hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,12 @@
6565
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
6666
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
6767
import org.apache.hadoop.hbase.regionserver.HStoreFile;
68+
import org.apache.hadoop.hbase.regionserver.StoreContext;
6869
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
6970
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
7071
import org.apache.hadoop.hbase.regionserver.TestHStoreFile;
72+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
73+
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
7174
import org.apache.hadoop.hbase.testclassification.IOTests;
7275
import org.apache.hadoop.hbase.testclassification.MediumTests;
7376
import org.apache.hadoop.hbase.trace.TraceUtil;
@@ -357,11 +360,14 @@ private void testPrefetchWhenRefs(boolean compactionEnabled, Consumer<Cacheable>
357360
Path storeFile = fileWithSplitPoint.getFirst();
358361
HRegionFileSystem regionFS =
359362
HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, region);
360-
HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true);
363+
StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true,
364+
StoreContext.getBuilder().withFamilyStoreDirectoryPath(new Path(regionDir, "cf"))
365+
.withRegionFileSystem(regionFS).build());
366+
HStoreFile file = new HStoreFile(fs, storeFile, conf, cacheConf, BloomType.NONE, true, sft);
361367
Path ref = regionFS.splitStoreFile(region, "cf", file, fileWithSplitPoint.getSecond(), false,
362-
new ConstantSizeRegionSplitPolicy());
368+
new ConstantSizeRegionSplitPolicy(), sft);
363369
conf.setBoolean(HBASE_REGION_SERVER_ENABLE_COMPACTION, compactionEnabled);
364-
HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true);
370+
HStoreFile refHsf = new HStoreFile(this.fs, ref, conf, cacheConf, BloomType.NONE, true, sft);
365371
refHsf.initReader();
366372
HFile.Reader reader = refHsf.getReader().getHFileReader();
367373
while (!reader.prefetchComplete()) {
@@ -403,8 +409,12 @@ private void testPrefetchWhenHFileLink(Consumer<Cacheable> test) throws Exceptio
403409
Path linkFilePath =
404410
new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
405411

412+
StoreFileTracker sft = StoreFileTrackerFactory.create(conf, true,
413+
StoreContext.getBuilder()
414+
.withFamilyStoreDirectoryPath(new Path(regionFs.getRegionDir(), "cf"))
415+
.withRegionFileSystem(regionFs).build());
406416
// Try to open store file from link
407-
StoreFileInfo storeFileInfo = new StoreFileInfo(testConf, this.fs, linkFilePath, true);
417+
StoreFileInfo storeFileInfo = sft.getStoreFileInfo(linkFilePath, true);
408418
HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);
409419
assertTrue(storeFileInfo.isLink());
410420

0 commit comments

Comments
 (0)