From d081df9b4adda39aeeac29b5dd9a4d15179ab2aa Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Fri, 8 Jun 2018 21:58:41 -0700 Subject: [PATCH 1/7] Add IFileSystem api to check if uri is a directory SKIP e2e tests address cr comments handle path not found expception cr comments fix address cr comments fix cr comments cr comment fixes fix validate children test --- .../TestAzureBlockBlobFileSystemE2E.cs | 69 +++++++++++-------- .../TestAzureDataLakeFileSystemE2E.cs | 14 ++++ .../TestHadoopFileSystem.cs | 41 +++++++++-- .../TestLocalFileSystem.cs | 16 +++++ .../AzureBlob/AzureBlockBlobFileSystem.cs | 15 +++- .../AzureBlob/AzureCloudBlobClient.cs | 21 ++++++ .../FileSystem/AzureBlob/ICloudBlobClient.cs | 14 +++- .../AzureDataLake/AzureDataLakeFileSystem.cs | 25 ++++++- .../FileSystem/Hadoop/HadoopFileSystem.cs | 11 +++ .../FileSystem/IFileSystem.cs | 7 ++ .../FileSystem/Local/LocalFileSystem.cs | 5 ++ 11 files changed, 198 insertions(+), 40 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs index f866944179..81eaead1f1 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs @@ -20,6 +20,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Linq; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Org.Apache.REEF.IO.FileSystem; @@ -161,9 +162,9 @@ public void TestGetChildBlobsInContainerE2E() [Fact(Skip = SkipMessage)] public void TestGetChildContainerInStorageAccountE2E() { - // List containers in the storage account - Uri rootUri = _fileSystem.CreateUriForPath(string.Empty); - ValidateChildren(rootUri, new List { _container.Uri }); + // List container uris in the storage account + var containerUris = _client.ListContainers().Select(container => container.Uri); + ValidateChildren(_client.BaseUri, containerUris); } private void ValidateChildren(Uri storageBlobUri, IEnumerable expectedChildBlobs) @@ -264,6 +265,21 @@ public void TestCopyFromLocalE2E() container.DeleteIfExistsAsync().Wait(); } + [Fact(Skip = SkipMessage)] + public void TestIsDirectory() + { + const string Directory1 = "dir1"; + const string Directory2 = "dir1/dir2"; + const string Directory3 = "dir3"; + var blockBlobs1 = CreateTempBlobs(Directory1); + CreateTempBlobs(Directory2); + + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory1))); + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory2))); + Assert.False(_fileSystem.IsDirectory(PathToFile(Directory3))); + Assert.False(_fileSystem.IsDirectory(blockBlobs1.First().Uri)); + } + [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryAtContainerE2E() { @@ -274,16 +290,8 @@ public void TestDeleteDirectoryAtContainerE2E() [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryFirstLevelE2E() { - const string Directory = "dir"; - var blockBlobs = new List(); - for (var i = 0; i < 3; i++) - { - var filePath = Directory + '/' + i; - var blockBlob = _container.GetBlockBlobReference(filePath); - UploadFromString(blockBlob, "hello"); - Assert.True(CheckBlobExists(blockBlob)); - blockBlobs.Add(blockBlob); - } + const string Directory = "dir1"; + var blockBlobs = CreateTempBlobs(Directory); _fileSystem.DeleteDirectory(PathToFile(Directory)); @@ -299,24 +307,11 @@ public void TestDeleteDirectoryFirstLevelE2E() public void TestDeleteDirectorySecondLevelE2E() { const string Directory1 = "dir1"; - const string Directory2 = "dir2"; - var blockBlobs1 = new List(); - var blockBlobs2 = new List(); - for (var i = 0; i < 3; i++) - { - var filePath1 = Directory1 + '/' + i; - var filePath2 = Directory1 + '/' + Directory2 + '/' + i; - var blockBlob1 = _container.GetBlockBlobReference(filePath1); - var blockBlob2 = _container.GetBlockBlobReference(filePath2); - UploadFromString(blockBlob1, "hello"); - UploadFromString(blockBlob2, "hello"); - Assert.True(CheckBlobExists(blockBlob1)); - Assert.True(CheckBlobExists(blockBlob2)); - blockBlobs1.Add(blockBlob1); - blockBlobs2.Add(blockBlob2); - } + const string Directory2 = "dir1/dir2"; + var blockBlobs1 = CreateTempBlobs(Directory1); + var blockBlobs2 = CreateTempBlobs(Directory2); - _fileSystem.DeleteDirectory(PathToFile(Directory1 + '/' + Directory2)); + _fileSystem.DeleteDirectory(PathToFile(Directory2)); foreach (var blockBlob in blockBlobs2) { @@ -331,6 +326,20 @@ public void TestDeleteDirectorySecondLevelE2E() Assert.True(CheckContainerExists(_container)); } + private List CreateTempBlobs(string directory, int fileCount = 3) + { + var blockBlobs = new List(); + for (var i = 0; i < fileCount; i++) + { + var filePath = directory + '/' + i; + var blockBlob = _container.GetBlockBlobReference(filePath); + UploadFromString(blockBlob, "hello"); + Assert.True(CheckBlobExists(blockBlob)); + blockBlobs.Add(blockBlob); + } + return blockBlobs; + } + private static void UploadFromString(ICloudBlob blob, string str) { var byteArray = Encoding.UTF8.GetBytes(str); diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs index 131c941f76..f139e3df2d 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs @@ -194,6 +194,20 @@ public void TestCreateDirectoryE2E() Assert.True(_adlsClient.CheckExists(dirName)); } + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryE2E() + { + string dirName = $"/{_defaultFolderName}"; + string fakeDirName = $"/fakeDir"; + + _adlsClient.CreateDirectory(dirName); + string fileName = UploadFromString(ContentsText); + + Assert.True(_fileSystem.IsDirectory(PathToFile(dirName))); + Assert.False(_fileSystem.IsDirectory(PathToFile(fakeDirName))); + Assert.False(_fileSystem.IsDirectory(PathToFile(fileName))); + } + [Fact(Skip = SkipMessage)] public void TestDeleteDirectoryE2E() { diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs index 94297a55e8..0de6dd7c9e 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs @@ -33,6 +33,7 @@ namespace Org.Apache.REEF.IO.Tests /// public sealed class TestHadoopFileSystem { + private const string SkipMessage = "These tests need to be run in an environment with HDFS installed."; // Use null to run tests private HadoopFileSystem _fileSystem; private Uri GetTempUri() @@ -56,7 +57,7 @@ public TestHadoopFileSystem() /// /// Creates a temp file locally, uploads it to HDFS and downloads it again. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestCopyFromLocalAndBack() { var localFile = FileSystemTestUtilities.MakeLocalTempFile(); @@ -77,7 +78,7 @@ public void TestCopyFromLocalAndBack() /// /// Tests whether .Exists() works. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestExists() { var remoteUri = GetTempUri(); @@ -93,7 +94,7 @@ public void TestExists() /// /// Tests for .GetChildren(). /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestGetChildren() { // Make a directory @@ -129,14 +130,14 @@ public void TestGetChildren() _fileSystem.DeleteDirectory(remoteDirectory); } - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestOpen() { // Open() is not supported by HadoopFileSystem. Use CopyToLocal and open the local file instead. Assert.Throws(() => _fileSystem.Open(GetTempUri())); } - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestCreate() { // Create() is not supported by HadoopFileSystem. Create a local file and use CopyFromLocal instead. @@ -163,10 +164,38 @@ public void CreateUriForPathWithPrefix() Assert.Equal(new Uri(uriString), _fileSystem.CreateUriForPath(uriString)); } + /// + /// Tests whether .IsDirectory() works. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectory() + { + // Create directory + var remoteDirUri = GetTempUri(); + _fileSystem.CreateDirectory(remoteDirUri); + + // Create fake directory uri + var remoteFakeuri = GetTempUri(); + + // Create file + var remoteFileUri = GetTempUri(); + var localFile = FileSystemTestUtilities.MakeLocalTempFile(); + _fileSystem.CopyFromLocal(localFile, remoteFileUri); + + Assert.True(_fileSystem.IsDirectory(remoteDirUri)); + Assert.False(_fileSystem.IsDirectory(remoteFakeuri)); + Assert.False(_fileSystem.IsDirectory(remoteFileUri)); + + // Clean up + _fileSystem.DeleteDirectory(remoteDirUri); + _fileSystem.Delete(remoteFileUri); + File.Delete(localFile); + } + /// /// This test is to make sure with the HadoopFileSystemConfiguration, HadoopFileSystem can be injected. /// - [Fact(Skip = "These tests need to be run in an environment with HDFS installed.")] + [Fact(Skip = SkipMessage)] public void TestHadoopFileSystemConfiguration() { var fileSystemTest = TangFactory.GetTang().NewInjector(HadoopFileSystemConfiguration.ConfigurationModule diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs index 6272cb603a..396858e09d 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -136,6 +136,22 @@ public void TestCopy() Assert.False(fs.Exists(sourceUri)); } + [Fact] + public void TestIsDirectory() + { + var fs = GetFileSystem(); + var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName) + "/"); + var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir") + "/"); + fs.CreateDirectory(directoryUri); + var fileUri = new Uri(directoryUri, "testfile"); + MakeRemoteTestFile(fs, fileUri); + Assert.True(fs.IsDirectory(directoryUri)); + Assert.False(fs.IsDirectory(fakeDirectoryUri)); + Assert.False(fs.IsDirectory(fileUri)); + fs.Delete(fileUri); + fs.DeleteDirectory(directoryUri); + } + [Fact] public void TestGetChildren() { diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs index d1004c387c..6f1f7a1c6b 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs @@ -125,6 +125,19 @@ public void CreateDirectory(Uri directoryUri) { } + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + public bool IsDirectory(Uri uri) + { + var path = uri.AbsolutePath.TrimStart('/'); + const int maxBlobResults = 1; + var blobItems = _client.ListBlobsSegmented(path, false, BlobListingDetails.Metadata, maxBlobResults, null, null, null).Results; + return blobItems.Any() && blobItems.First() is CloudBlobDirectory; + } + /// /// Recursively deletes blobs under a specified "directory URI." /// If only the container is specified, the entire container is deleted. @@ -197,7 +210,7 @@ public IEnumerable GetChildren(Uri directoryUri) do { - BlobResultSegment listing = _client.ListBlobsSegmented( + BlobResultSegment listing = _client.ListDirectoryBlobsSegmented( containerName, relativeAddress, useFlatListing: false, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs index eec480673c..05d77e81a0 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs @@ -70,6 +70,27 @@ public ICloudBlockBlob GetBlockBlobReference(Uri uri) } public BlobResultSegment ListBlobsSegmented( + string prefix, + bool useFlatListing, + BlobListingDetails blobListingDetails, + int? maxResults, + BlobContinuationToken continuationToken, + BlobRequestOptions blobRequestOptions, + OperationContext operationContext) + { + var task = _client.ListBlobsSegmentedAsync( + prefix, + useFlatListing, + blobListingDetails, + maxResults, + continuationToken, + blobRequestOptions, + operationContext); + task.Wait(); + return task.Result; + } + + public BlobResultSegment ListDirectoryBlobsSegmented( string containerName, string relativeAddress, bool useFlatListing, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs index f2a43aa7d4..24492d036f 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs @@ -56,9 +56,21 @@ internal interface ICloudBlobClient ICloudBlockBlob GetBlockBlobReference(Uri uri); /// - /// Paginates a blob listing with container and relative path. + /// Paginates a blob listing with prefix. /// BlobResultSegment ListBlobsSegmented( + string path, + bool useFlatListing, + BlobListingDetails blobListingDetails, + int? maxResults, + BlobContinuationToken continuationToken, + BlobRequestOptions blobRequestOptions, + OperationContext operationContext); + + /// + /// Paginates a blob listing with container and relative path. + /// + BlobResultSegment ListDirectoryBlobsSegmented( string containerName, string relativeAddress, bool useFlatListing, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs index d1ad20aa27..7e14e5ae74 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs @@ -19,6 +19,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Text; using Microsoft.Azure.DataLake.Store; using Microsoft.Azure.DataLake.Store.FileTransfer; @@ -178,14 +179,34 @@ public void CreateDirectory(Uri directoryUri) } } + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + public bool IsDirectory(Uri uri) + { + try + { + return _adlsClient.GetDirectoryEntry(uri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY; + } + catch (AdlsException e) + { + if (e.HttpStatus == HttpStatusCode.NotFound) + { + return false; + } + throw e; + } + } + /// /// Deletes a directory. /// /// If directory cannot be deleted public void DeleteDirectory(Uri directoryUri) { - bool deleteStatus = Exists(directoryUri) && - _adlsClient.GetDirectoryEntry(directoryUri.AbsolutePath).Type == DirectoryEntryType.DIRECTORY && + bool deleteStatus = IsDirectory(directoryUri) && _adlsClient.DeleteRecursive(directoryUri.AbsolutePath); if (!deleteStatus) { diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs index cd80b629ab..3d0e6a5004 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs @@ -152,6 +152,17 @@ public void CreateDirectory(Uri directoryUri) _commandRunner.Run("dfs -mkdir " + directoryUri); } + public bool IsDirectory(Uri uri) + { + // TODO[JIRA REEF - 2039]: HadoopFileSystem .IsDirectory() check needs to work on linux machines. + var stdOut = _commandRunner.Run("dfs -test -d " + uri + "& call echo %^errorlevel%").StdOut; + if (stdOut.Any()) + { + return "0".Equals(stdOut.First()); + } + return false; + } + public void DeleteDirectory(Uri directoryUri) { _commandRunner.Run("dfs -rmdir " + directoryUri); diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs index 41e12cdde5..7d3b98318c 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/IFileSystem.cs @@ -90,6 +90,13 @@ public interface IFileSystem /// void CreateDirectory(Uri directoryUri); + /// + /// Checks if uri is a directory uri. + /// + /// uri of the directory/file + /// true if uri is for a directory else false + bool IsDirectory(Uri uri); + /// /// Deletes a directory. /// diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs index 313fdd9fb7..ca5a7efedb 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Local/LocalFileSystem.cs @@ -105,6 +105,11 @@ public void CreateDirectory(Uri directoryUri) Directory.CreateDirectory(directoryUri.LocalPath); } + public bool IsDirectory(Uri uri) + { + return Directory.Exists(uri.LocalPath); + } + public void DeleteDirectory(Uri directoryUri) { Directory.Delete(directoryUri.LocalPath, true); From 5d8189ae7f54f9dc90e494c52a2f7d4bee5ccc87 Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Tue, 26 Jun 2018 17:12:12 -0700 Subject: [PATCH 2/7] address pr comments --- .../TestAzureBlockBlobFileSystemE2E.cs | 4 +++- .../FileSystem/AzureBlob/AzureCloudBlobClient.cs | 1 - .../FileSystem/Hadoop/HadoopFileSystem.cs | 7 +------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs index 81eaead1f1..80f390a48f 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs @@ -36,7 +36,9 @@ namespace Org.Apache.REEF.IO.Tests /// public sealed class TestAzureBlockBlobFileSystemE2E : IDisposable { - private const string SkipMessage = "Fill in credentials before running test"; // Use null to run tests + // Uncomment SkipMessage = null to run tests + private const string SkipMessage = "Fill in credentials before running test"; + // private const string SkipMessage = null; private const string HelloFile = "hello"; private IFileSystem _fileSystem; private CloudBlobClient _client; diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs index 05d77e81a0..8a72e22b0e 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs @@ -86,7 +86,6 @@ public BlobResultSegment ListBlobsSegmented( continuationToken, blobRequestOptions, operationContext); - task.Wait(); return task.Result; } diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs index 3d0e6a5004..cff6cb1d6c 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Text.RegularExpressions; @@ -156,11 +155,7 @@ public bool IsDirectory(Uri uri) { // TODO[JIRA REEF - 2039]: HadoopFileSystem .IsDirectory() check needs to work on linux machines. var stdOut = _commandRunner.Run("dfs -test -d " + uri + "& call echo %^errorlevel%").StdOut; - if (stdOut.Any()) - { - return "0".Equals(stdOut.First()); - } - return false; + return stdOut.Any() && stdOut.First() == "0"; } public void DeleteDirectory(Uri directoryUri) From 272feaa0c7315ad47e8a36615d8a610518d69658 Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Wed, 27 Jun 2018 11:52:05 -0700 Subject: [PATCH 3/7] clean up --- .../Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs index 396858e09d..2dd9e76ac5 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -140,14 +140,21 @@ public void TestCopy() public void TestIsDirectory() { var fs = GetFileSystem(); + // Directory check var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName) + "/"); - var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir") + "/"); fs.CreateDirectory(directoryUri); + Assert.True(fs.IsDirectory(directoryUri)); + + // File check var fileUri = new Uri(directoryUri, "testfile"); MakeRemoteTestFile(fs, fileUri); - Assert.True(fs.IsDirectory(directoryUri)); - Assert.False(fs.IsDirectory(fakeDirectoryUri)); Assert.False(fs.IsDirectory(fileUri)); + + // Fake directory check + var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir") + "/"); + Assert.False(fs.IsDirectory(fakeDirectoryUri)); + + // Clean up fs.Delete(fileUri); fs.DeleteDirectory(directoryUri); } From e7f44e318bf6993edd85f631a7e88403b7c075be Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Fri, 29 Jun 2018 17:39:59 -0700 Subject: [PATCH 4/7] address PR comments --- .../TestAzureBlockBlobFileSystemE2E.cs | 60 +++++++++++-------- .../TestAzureDataLakeFileSystemE2E.cs | 18 ++++-- .../TestHadoopFileSystem.cs | 28 +++++++-- .../AzureBlob/AzureBlockBlobFileSystem.cs | 12 +++- .../AzureBlob/AzureCloudBlobClient.cs | 8 +-- .../AzureBlob/AzureCloudBlobContainer.cs | 3 +- .../AzureBlob/AzureCloudBlobDirectory.cs | 9 ++- .../AzureBlob/AzureCloudBlockBlob.cs | 11 ++-- .../FileSystem/AzureBlob/ICloudBlobClient.cs | 2 +- .../AzureDataLake/AzureDataLakeFileSystem.cs | 5 +- 10 files changed, 99 insertions(+), 57 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs index 80f390a48f..e798ecd6e8 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs @@ -69,26 +69,22 @@ public void Dispose() private bool CheckBlobExists(ICloudBlob blob) { - var task = blob.ExistsAsync(); - return task.Result; + return blob.ExistsAsync().GetAwaiter().GetResult(); } private bool CheckContainerExists(CloudBlobContainer container) { - var task = container.ExistsAsync(); - return task.Result; + return container.ExistsAsync().GetAwaiter().GetResult(); } private ICloudBlob GetBlobReferenceFromServer(CloudBlobContainer container, string blobName) { - var task = container.GetBlobReferenceFromServerAsync(blobName); - return task.Result; + return container.GetBlobReferenceFromServerAsync(blobName).GetAwaiter().GetResult(); } private string DownloadText(CloudBlockBlob blob) { - var task = blob.DownloadTextAsync(); - return task.Result; + return blob.DownloadTextAsync().GetAwaiter().GetResult(); } [Fact(Skip = SkipMessage)] @@ -268,18 +264,34 @@ public void TestCopyFromLocalE2E() } [Fact(Skip = SkipMessage)] - public void TestIsDirectory() + public void TestIsDirectoryValidDirectoryLevel1E2E() { - const string Directory1 = "dir1"; - const string Directory2 = "dir1/dir2"; - const string Directory3 = "dir3"; - var blockBlobs1 = CreateTempBlobs(Directory1); - CreateTempBlobs(Directory2); + const string Directory = "dir"; + CreateTempBlobs(Directory); + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory))); + } - Assert.True(_fileSystem.IsDirectory(PathToFile(Directory1))); - Assert.True(_fileSystem.IsDirectory(PathToFile(Directory2))); - Assert.False(_fileSystem.IsDirectory(PathToFile(Directory3))); - Assert.False(_fileSystem.IsDirectory(blockBlobs1.First().Uri)); + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryValidDirectoryLevel2E2E() + { + const string Directory = "dir1/dir2"; + CreateTempBlobs(Directory); + Assert.True(_fileSystem.IsDirectory(PathToFile(Directory))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectoryE2E() + { + const string Directory = "dir"; + Assert.False(_fileSystem.IsDirectory(PathToFile(Directory))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFileE2E() + { + const string Directory = "dir"; + var blockBlobs = CreateTempBlobs(Directory); + Assert.False(_fileSystem.IsDirectory(blockBlobs.First().Uri)); } [Fact(Skip = SkipMessage)] @@ -328,18 +340,16 @@ public void TestDeleteDirectorySecondLevelE2E() Assert.True(CheckContainerExists(_container)); } - private List CreateTempBlobs(string directory, int fileCount = 3) + private IEnumerable CreateTempBlobs(string directory, int fileCount = 3) { - var blockBlobs = new List(); - for (var i = 0; i < fileCount; i++) + return Enumerable.Range(0, fileCount).Select(i => { var filePath = directory + '/' + i; var blockBlob = _container.GetBlockBlobReference(filePath); UploadFromString(blockBlob, "hello"); - Assert.True(CheckBlobExists(blockBlob)); - blockBlobs.Add(blockBlob); - } - return blockBlobs; + Assert.True(CheckBlobExists(blockBlob), "Blob does not exist: " + filePath); + return blockBlob; + }); } private static void UploadFromString(ICloudBlob blob, string str) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs index f139e3df2d..6d4841b699 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureDataLakeFileSystemE2E.cs @@ -195,16 +195,24 @@ public void TestCreateDirectoryE2E() } [Fact(Skip = SkipMessage)] - public void TestIsDirectoryE2E() + public void TestIsDirectoryValidDirectoryE2E() { string dirName = $"/{_defaultFolderName}"; - string fakeDirName = $"/fakeDir"; - _adlsClient.CreateDirectory(dirName); - string fileName = UploadFromString(ContentsText); + Assert.True(_fileSystem.IsDirectory(PathToFile(dirName))); + } - Assert.True(_fileSystem.IsDirectory(PathToFile(dirName))); + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectoryE2E() + { + string fakeDirName = $"/fakeDir"; Assert.False(_fileSystem.IsDirectory(PathToFile(fakeDirName))); + } + + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFileE2E() + { + string fileName = UploadFromString(ContentsText); Assert.False(_fileSystem.IsDirectory(PathToFile(fileName))); } diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs index 0de6dd7c9e..7872bc8286 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestHadoopFileSystem.cs @@ -165,29 +165,47 @@ public void CreateUriForPathWithPrefix() } /// - /// Tests whether .IsDirectory() works. + /// Tests whether .IsDirectory() works for valid directory. /// [Fact(Skip = SkipMessage)] - public void TestIsDirectory() + public void TestIsDirectoryValidDirectory() { // Create directory var remoteDirUri = GetTempUri(); _fileSystem.CreateDirectory(remoteDirUri); + Assert.True(_fileSystem.IsDirectory(remoteDirUri)); + + // Clean up + _fileSystem.DeleteDirectory(remoteDirUri); + } + + /// + /// Tests whether .IsDirectory() works for fake directory. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFakeDirectory() + { // Create fake directory uri var remoteFakeuri = GetTempUri(); + Assert.False(_fileSystem.IsDirectory(remoteFakeuri)); + } + + /// + /// Tests whether .IsDirectory() works for file. + /// + [Fact(Skip = SkipMessage)] + public void TestIsDirectoryFile() + { // Create file var remoteFileUri = GetTempUri(); var localFile = FileSystemTestUtilities.MakeLocalTempFile(); _fileSystem.CopyFromLocal(localFile, remoteFileUri); - Assert.True(_fileSystem.IsDirectory(remoteDirUri)); - Assert.False(_fileSystem.IsDirectory(remoteFakeuri)); Assert.False(_fileSystem.IsDirectory(remoteFileUri)); // Clean up - _fileSystem.DeleteDirectory(remoteDirUri); _fileSystem.Delete(remoteFileUri); File.Delete(localFile); } diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs index 6f1f7a1c6b..6b63465d2d 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs @@ -133,9 +133,15 @@ public void CreateDirectory(Uri directoryUri) public bool IsDirectory(Uri uri) { var path = uri.AbsolutePath.TrimStart('/'); - const int maxBlobResults = 1; - var blobItems = _client.ListBlobsSegmented(path, false, BlobListingDetails.Metadata, maxBlobResults, null, null, null).Results; - return blobItems.Any() && blobItems.First() is CloudBlobDirectory; + var blobItems = _client.ListBlobsSegmented( + path, + useFlatListing: false, + BlobListingDetails.Metadata, + maxResults: 1, + continuationToken: null, + blobRequestOptions: null, + operationContext: null).Results; + return blobItems.OfType().Any(); } /// diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs index 8a72e22b0e..72ed015b84 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobClient.cs @@ -55,8 +55,7 @@ public Uri BaseUri public ICloudBlob GetBlobReferenceFromServer(Uri blobUri) { - var task = _client.GetBlobReferenceFromServerAsync(blobUri); - return task.Result; + return _client.GetBlobReferenceFromServerAsync(blobUri).GetAwaiter().GetResult(); } public ICloudBlobContainer GetContainerReference(string containerName) @@ -78,15 +77,14 @@ public BlobResultSegment ListBlobsSegmented( BlobRequestOptions blobRequestOptions, OperationContext operationContext) { - var task = _client.ListBlobsSegmentedAsync( + return _client.ListBlobsSegmentedAsync( prefix, useFlatListing, blobListingDetails, maxResults, continuationToken, blobRequestOptions, - operationContext); - return task.Result; + operationContext).GetAwaiter().GetResult(); } public BlobResultSegment ListDirectoryBlobsSegmented( diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs index e2fbd6161d..53ae2aca14 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs @@ -35,8 +35,7 @@ public AzureCloudBlobContainer(CloudBlobContainer container, BlobRequestOptions public bool CreateIfNotExists() { - var task = _container.CreateIfNotExistsAsync(_requestOptions, null); - return task.Result; + return _container.CreateIfNotExistsAsync(_requestOptions, null).GetAwaiter().GetResult(); } public void DeleteIfExists() diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs index 4d3295e637..43963084fd 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobDirectory.cs @@ -39,8 +39,13 @@ public ICloudBlobDirectory GetDirectoryReference(string directoryName) public IEnumerable ListBlobs(bool useFlatListing = false) { - var task = _directory.ListBlobsSegmentedAsync(useFlatListing, BlobListingDetails.All, null, null, null, null); - return task.Result.Results; + return _directory.ListBlobsSegmentedAsync( + useFlatBlobListing: useFlatListing, + BlobListingDetails.All, + maxResults: null, + null, + null, + null).GetAwaiter().GetResult().Results; } } } diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs index c8835687d1..26e9ec9fdf 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs @@ -67,7 +67,7 @@ public Stream Open() { #if REEF_DOTNET_BUILD var task = _blob.OpenReadAsync(null, _requestOptions, null); - return task.Result; + return task.GetAwaiter().GetResult(); #else return _blob.OpenRead(null, _requestOptions, null); #endif @@ -76,8 +76,7 @@ public Stream Open() public Stream Create() { #if REEF_DOTNET_BUILD - var task = _blob.OpenWriteAsync(null, _requestOptions, null); - return task.Result; + return _blob.OpenWriteAsync(null, _requestOptions, null).GetAwaiter().GetResult(); #else return _blob.OpenWrite(null, _requestOptions, null); #endif @@ -85,8 +84,7 @@ public Stream Create() public bool Exists() { - var task = _blob.ExistsAsync(_requestOptions, null); - return task.Result; + return _blob.ExistsAsync(_requestOptions, null).GetAwaiter().GetResult(); } public void Delete() @@ -101,8 +99,7 @@ public void DeleteIfExists() public string StartCopy(Uri source) { - var task = _blob.StartCopyAsync(source, null, null, _requestOptions, null); - return task.Result; + return _blob.StartCopyAsync(source, null, null, _requestOptions, null).GetAwaiter().GetResult(); } public void DownloadToFile(string path, FileMode mode) diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs index 24492d036f..d4ce481177 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/ICloudBlobClient.cs @@ -59,7 +59,7 @@ internal interface ICloudBlobClient /// Paginates a blob listing with prefix. /// BlobResultSegment ListBlobsSegmented( - string path, + string prefix, bool useFlatListing, BlobListingDetails blobListingDetails, int? maxResults, diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs index 7e14e5ae74..b0f250c0d8 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureDataLake/AzureDataLakeFileSystem.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Net; @@ -249,10 +250,10 @@ public Uri CreateUriForPath(string path) } catch (UriFormatException) { - resultUri = new Uri(new Uri(this.UriPrefix), path); + resultUri = new Uri(new Uri(UriPrefix), path); } - if (!resultUri.AbsoluteUri.StartsWith(this.UriPrefix)) + if (!resultUri.AbsoluteUri.StartsWith(UriPrefix, true, CultureInfo.InvariantCulture)) { throw new ArgumentException($"Given URI must begin with valid prefix ({this.UriPrefix})", nameof(path)); } From ff4ad028cf4104de2bca4c4ea37dbeeace104da5 Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Fri, 29 Jun 2018 17:53:00 -0700 Subject: [PATCH 5/7] Address PR comments --- .../TestAzureBlockBlobFileSystemE2E.cs | 7 +++---- .../cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs | 10 +++++----- .../FileSystem/Hadoop/HadoopFileSystem.cs | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs index e798ecd6e8..9cd5d32576 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs @@ -20,7 +20,6 @@ using System.IO; using System.Linq; using System.Text; -using System.Linq; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Org.Apache.REEF.IO.FileSystem; @@ -40,9 +39,9 @@ public sealed class TestAzureBlockBlobFileSystemE2E : IDisposable private const string SkipMessage = "Fill in credentials before running test"; // private const string SkipMessage = null; private const string HelloFile = "hello"; - private IFileSystem _fileSystem; - private CloudBlobClient _client; - private CloudBlobContainer _container; + private readonly IFileSystem _fileSystem; + private readonly CloudBlobClient _client; + private readonly CloudBlobContainer _container; public TestAzureBlockBlobFileSystemE2E() { diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs index 2dd9e76ac5..a2265819bd 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -141,18 +141,18 @@ public void TestIsDirectory() { var fs = GetFileSystem(); // Directory check - var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName) + "/"); + var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName)); fs.CreateDirectory(directoryUri); - Assert.True(fs.IsDirectory(directoryUri)); + Assert.True(fs.IsDirectory(directoryUri), ".IsDirectory() failed on: " + directoryUri); // File check var fileUri = new Uri(directoryUri, "testfile"); MakeRemoteTestFile(fs, fileUri); - Assert.False(fs.IsDirectory(fileUri)); + Assert.False(fs.IsDirectory(fileUri), ".IsDirectory() failed on: " + fileUri); // Fake directory check - var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir") + "/"); - Assert.False(fs.IsDirectory(fakeDirectoryUri)); + var fakeDirectoryUri = new Uri(Path.Combine(Path.GetTempPath(), "fakeDir")); + Assert.False(fs.IsDirectory(fakeDirectoryUri), ".IsDirectory() failed on: " + fakeDirectoryUri); // Clean up fs.Delete(fileUri); diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs index cff6cb1d6c..4f6e1cc183 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/Hadoop/HadoopFileSystem.cs @@ -155,7 +155,7 @@ public bool IsDirectory(Uri uri) { // TODO[JIRA REEF - 2039]: HadoopFileSystem .IsDirectory() check needs to work on linux machines. var stdOut = _commandRunner.Run("dfs -test -d " + uri + "& call echo %^errorlevel%").StdOut; - return stdOut.Any() && stdOut.First() == "0"; + return stdOut.FirstOrDefault() == "0"; } public void DeleteDirectory(Uri directoryUri) From 7e443a691f1d210b5f657d84e7e92cb5ef324dea Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Fri, 29 Jun 2018 18:00:33 -0700 Subject: [PATCH 6/7] Address pr comments --- .../FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs index 6b63465d2d..09f527c989 100644 --- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureBlockBlobFileSystem.cs @@ -132,7 +132,7 @@ public void CreateDirectory(Uri directoryUri) /// true if uri is for a directory else false public bool IsDirectory(Uri uri) { - var path = uri.AbsolutePath.TrimStart('/'); + var path = uri.AbsolutePath.TrimStart(UrlPathSeparator); var blobItems = _client.ListBlobsSegmented( path, useFlatListing: false, @@ -150,7 +150,7 @@ public bool IsDirectory(Uri uri) /// public void DeleteDirectory(Uri directoryUri) { - var uriSplit = directoryUri.AbsolutePath.Split(new[] { "/" }, StringSplitOptions.RemoveEmptyEntries); + var uriSplit = directoryUri.AbsolutePath.Split(new[] { UrlPathSeparator }, StringSplitOptions.RemoveEmptyEntries); if (!uriSplit.Any()) { throw new StorageException( From 23bf5948f07f4dead31519829e27b422e0e3adb1 Mon Sep 17 00:00:00 2001 From: Dwaipayan Mukherjee Date: Fri, 29 Jun 2018 18:12:01 -0700 Subject: [PATCH 7/7] Address pr comments --- lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs index a2265819bd..8e1e226f9a 100644 --- a/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs +++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestLocalFileSystem.cs @@ -141,7 +141,7 @@ public void TestIsDirectory() { var fs = GetFileSystem(); // Directory check - var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), TempFileName)); + var directoryUri = new Uri(Path.Combine(Path.GetTempPath(), "dir")); fs.CreateDirectory(directoryUri); Assert.True(fs.IsDirectory(directoryUri), ".IsDirectory() failed on: " + directoryUri);