Skip to content

HADOOP-1320. Dir Marker getFileStatus() changes backport #2170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

HADOOP-1320. Dir Marker getFileStatus() changes backport #2170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
12d33fa
HADOOP-1320 Dir Marker getFileStatus() changes backport
steveloughran Jul 24, 2020
6a29f10
HADOOP-13230 fix test failure
steveloughran Jul 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions ...common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractGetFileStatusTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;

import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand Down Expand Up @@ -402,7 +404,8 @@ private Path createDirWithEmptySubFolder() throws IOException {
Path path = getContract().getTestPath();
fs.delete(path, true);
// create a - non-qualified - Path for a subdir
Path subfolder = path.suffix('/' + this.methodName.getMethodName());
Path subfolder = path.suffix('/' + this.methodName.getMethodName()
+ "-" + UUID.randomUUID());
mkdirs(subfolder);
return subfolder;
}
Expand Down Expand Up @@ -527,7 +530,8 @@ private FileStatus[] verifyListStatus(int expected,
Path path,
PathFilter filter) throws IOException {
FileStatus[] result = getFileSystem().listStatus(path, filter);
assertEquals("length of listStatus(" + path + ", " + filter + " )",
assertEquals("length of listStatus(" + path + ", " + filter + " ) " +
Arrays.toString(result),
expected, result.length);
return result;
}
Expand Down
187 changes: 82 additions & 105 deletions hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.s3a.impl.ChangeDetectionPolicy;
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
Expand Down Expand Up @@ -2172,6 +2173,8 @@ S3AFileStatus innerGetFileStatus(final Path f,

FileStatus msStatus = pm.getFileStatus();
if (needEmptyDirectoryFlag && msStatus.isDirectory()) {
// the caller needs to know if a directory is empty,
// and that this is a directory.
if (pm.isEmptyDirectory() != Tristate.UNKNOWN) {
// We have a definitive true / false from MetadataStore, we are done.
return S3AFileStatus.fromFileStatus(msStatus, pm.isEmptyDirectory());
Expand All @@ -2180,28 +2183,33 @@ S3AFileStatus innerGetFileStatus(final Path f,
if (children != null) {
tombstones = children.listTombstones();
}
LOG.debug("MetadataStore doesn't know if dir is empty, using S3.");
LOG.debug("MetadataStore doesn't know if {} is empty, using S3.",
path);
}
} else {
// Either this is not a directory, or we don't care if it is empty
return S3AFileStatus.fromFileStatus(msStatus, pm.isEmptyDirectory());
}

// If the metadata store has no children for it and it's not listed in
// S3 yet, we'll assume the empty directory is true;
S3AFileStatus s3FileStatus;
// now issue the S3 getFileStatus call.
try {
s3FileStatus = s3GetFileStatus(path, key, tombstones);
S3AFileStatus s3FileStatus = s3GetFileStatus(path, key,
StatusProbeEnum.ALL,
tombstones,
true);
// entry was found, so save in S3Guard and return the final value.
return S3Guard.putAndReturn(metadataStore, s3FileStatus,
instrumentation);
} catch (FileNotFoundException e) {
return S3AFileStatus.fromFileStatus(msStatus, Tristate.TRUE);
}
// entry was found, save in S3Guard
return S3Guard.putAndReturn(metadataStore, s3FileStatus, instrumentation);
} else {
// there was no entry in S3Guard
// retrieve the data and update the metadata store in the process.
return S3Guard.putAndReturn(metadataStore,
s3GetFileStatus(path, key, tombstones), instrumentation);
s3GetFileStatus(path, key, StatusProbeEnum.ALL,
tombstones, needEmptyDirectoryFlag),
instrumentation);
}
}

Expand All @@ -2212,87 +2220,96 @@ S3AFileStatus innerGetFileStatus(final Path f,
* Retry policy: retry translated.
* @param path Qualified path
* @param key Key string for the path
* @param probes probes to make
* @param tombstones tombstones to filter
* @param needEmptyDirectoryFlag if true, implementation will calculate
* a TRUE or FALSE value for {@link S3AFileStatus#isEmptyDirectory()}
* @return Status
* @throws FileNotFoundException when the path does not exist
* @throws FileNotFoundException the supplied probes failed.
* @throws IOException on other problems.
*/
@VisibleForTesting
@Retries.RetryTranslated
private S3AFileStatus s3GetFileStatus(final Path path, String key,
Set<Path> tombstones) throws IOException {
if (!key.isEmpty()) {
S3AFileStatus s3GetFileStatus(final Path path,
final String key,
final Set<StatusProbeEnum> probes,
@Nullable final Set<Path> tombstones,
final boolean needEmptyDirectoryFlag) throws IOException {
LOG.debug("S3GetFileStatus {}", path);
Preconditions.checkArgument(!needEmptyDirectoryFlag
|| probes.contains(StatusProbeEnum.List),
"s3GetFileStatus(%s) wants to know if a directory is empty but"
+ " does not request a list probe", path);

if (!key.isEmpty() && !key.endsWith("/")
&& probes.contains(StatusProbeEnum.Head)) {
try {
// look for the simple file
ObjectMetadata meta = getObjectMetadata(key);

if (objectRepresentsDirectory(key, meta.getContentLength())) {
LOG.debug("Found exact file: fake directory");
return new S3AFileStatus(Tristate.TRUE, path, username);
} else {
LOG.debug("Found exact file: normal file");
LOG.debug("Found exact file: normal file {}", key);
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
path,
getDefaultBlockSize(path),
username);
}
} catch (AmazonServiceException e) {
// if the response is a 404 error, it just means that there is
// no file at that path...the remaining checks will be needed.
if (e.getStatusCode() != 404) {
throw translateException("getFileStatus", path, e);
}
} catch (AmazonClientException e) {
throw translateException("getFileStatus", path, e);
}
}

// Necessary?
if (!key.endsWith("/")) {
String newKey = key + "/";
// execute the list
if (probes.contains(StatusProbeEnum.List)) {
try {
ObjectMetadata meta = getObjectMetadata(newKey);

if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
LOG.debug("Found file (with /): fake directory");
return new S3AFileStatus(Tristate.TRUE, path, username);
// this will find a marker dir / as well as an entry.
// When making a simple "is this a dir check" all is good.
// but when looking for an empty dir, we need to verify there are no
// children, so ask for two entries, so as to find
// a child
String dirKey = maybeAddTrailingSlash(key);
// list size is dir marker + at least one non-tombstone entry
// there's a corner case: more tombstones than you have in a
// single page list. We assume that if you have been deleting
// that many files, then the AWS listing will have purged some
// by the time of listing so that the response includes some
// which have not.

int listSize;
if (tombstones == null) {
// no tombstones so look for a marker and at least one child.
listSize = 2;
} else {
LOG.warn("Found file (with /): real file? should not happen: {}",
key);

return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
path,
getDefaultBlockSize(path),
username);
}
} catch (AmazonServiceException e) {
if (e.getStatusCode() != 404) {
throw translateException("getFileStatus", newKey, e);
}
} catch (AmazonClientException e) {
throw translateException("getFileStatus", newKey, e);
}
// build a listing > tombstones. If the caller has many thousands
// of tombstones this won't work properly, which is why pruning
// of expired tombstones matters.
listSize = Math.min(2 + tombstones.size(), Math.max(2, maxKeys));
}
}

try {
key = maybeAddTrailingSlash(key);
S3ListRequest request = createListObjectsRequest(key, "/", 1);
S3ListRequest request = createListObjectsRequest(dirKey, "/",
listSize);
// execute the request
S3ListResult listResult = listObjects(request);

S3ListResult objects = listObjects(request);

Collection<String> prefixes = objects.getCommonPrefixes();
Collection<S3ObjectSummary> summaries = objects.getObjectSummaries();
if (!isEmptyOfKeys(prefixes, tombstones) ||
!isEmptyOfObjects(summaries, tombstones)) {
if (listResult.hasPrefixesOrObjects(this::keyToPath, tombstones)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found path as directory (with /): {}/{}",
prefixes.size(), summaries.size());

for (S3ObjectSummary summary : summaries) {
LOG.debug("Summary: {} {}", summary.getKey(), summary.getSize());
LOG.debug("Found path as directory (with /)");
listResult.logAtDebug(LOG);
}
for (String prefix : prefixes) {
LOG.debug("Prefix: {}", prefix);
// At least one entry has been found.
// If looking for an empty directory, the marker must exist but no children.
// So the listing must contain the marker entry only.
if (needEmptyDirectoryFlag
&& listResult.representsEmptyDirectory(
this::keyToPath, dirKey, tombstones)) {
return new S3AFileStatus(Tristate.TRUE, path, username);
}
}

// either an empty directory is not needed, or the
// listing does not meet the requirements.
return new S3AFileStatus(Tristate.FALSE, path, username);
} else if (key.isEmpty()) {
LOG.debug("Found root directory");
Expand All @@ -2305,53 +2322,12 @@ private S3AFileStatus s3GetFileStatus(final Path path, String key,
} catch (AmazonClientException e) {
throw translateException("getFileStatus", path, e);
}
}

LOG.debug("Not Found: {}", path);
throw new FileNotFoundException("No such file or directory: " + path);
}

/**
* Helper function to determine if a collection of paths is empty
* after accounting for tombstone markers (if provided).
* @param keys Collection of path (prefixes / directories or keys).
* @param tombstones Set of tombstone markers, or null if not applicable.
* @return false if summaries contains objects not accounted for by
* tombstones.
*/
private boolean isEmptyOfKeys(Collection<String> keys, Set<Path>
tombstones) {
if (tombstones == null) {
return keys.isEmpty();
}
for (String key : keys) {
Path qualified = keyToQualifiedPath(key);
if (!tombstones.contains(qualified)) {
return false;
}
}
return true;
}

/**
* Helper function to determine if a collection of object summaries is empty
* after accounting for tombstone markers (if provided).
* @param summaries Collection of objects as returned by listObjects.
* @param tombstones Set of tombstone markers, or null if not applicable.
* @return false if summaries contains objects not accounted for by
* tombstones.
*/
private boolean isEmptyOfObjects(Collection<S3ObjectSummary> summaries,
Set<Path> tombstones) {
if (tombstones == null) {
return summaries.isEmpty();
}
Collection<String> stringCollection = new ArrayList<>(summaries.size());
for (S3ObjectSummary summary : summaries) {
stringCollection.add(summary.getKey());
}
return isEmptyOfKeys(stringCollection, tombstones);
}

/**
* Raw version of {@link FileSystem#exists(Path)} which uses S3 only:
* S3Guard MetadataStore, if any, will be skipped.
Expand All @@ -2364,7 +2340,8 @@ private boolean s3Exists(final Path f) throws IOException {
Path path = qualify(f);
String key = pathToKey(path);
try {
s3GetFileStatus(path, key, null);
s3GetFileStatus(path, key, StatusProbeEnum.ALL,
null, false);
return true;
} catch (FileNotFoundException e) {
return false;
Expand Down
Loading