Skip to content

Commit 3aa364e

Browse files
bluewwblueww
authored
[Storage] Support DatalakeGen2 SAS token (#18110)
* [Storage] Support DatalakeGen2 SAS token * fix CI failure * fix review comments
1 parent 7d4bee9 commit 3aa364e

File tree

12 files changed

+524
-14
lines changed

12 files changed

+524
-14
lines changed

‎src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -846,7 +846,9 @@ function Test-DatalakeGen2
846846
-Permission rw-rw--wx `
847847
-Owner '$superuser' `
848848
-Group '$superuser'
849-
$file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1
849+
$sas = New-AzDataLakeGen2SasToken -FileSystem $filesystemName -Path $filePath1 -Permission rw -Context $storageContext
850+
$ctxsas = New-AzStorageContext -StorageAccountName $StorageAccountName -SasToken $sas
851+
$file1 = Get-AzDataLakeGen2Item -Context $ctxsas -FileSystem $filesystemName -Path $filePath1
850852
Assert-AreEqual $file1.Path $filePath1
851853
Assert-AreEqual $file1.Permissions.ToSymbolicPermissions() "rw-rw--wx"
852854
Assert-AreEqual $file1.Properties.ContentType $ContentType

‎src/Storage/Storage.Management.Test/Storage.Management.Test.csproj‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14-
<PackageReference Include="Azure.Storage.Blobs" Version="12.11.0" />
15-
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.9.0" />
16-
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.9.0" />
17-
<PackageReference Include="Azure.Storage.Queues" Version="12.9.0" />
14+
<PackageReference Include="Azure.Storage.Blobs" Version="12.12.0" />
15+
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.10.0" />
16+
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.10.0" />
17+
<PackageReference Include="Azure.Storage.Queues" Version="12.10.0" />
1818
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="24.0.0" />
1919
</ItemGroup>
2020

‎src/Storage/Storage.Management/Az.Storage.psd1‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,8 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey',
197197
'Set-AzStorageBlobImmutabilityPolicy',
198198
'Remove-AzStorageBlobImmutabilityPolicy',
199199
'Set-AzStorageBlobLegalHold',
200-
'Invoke-AzRmStorageContainerImmutableStorageWithVersioningMigration'
200+
'Invoke-AzRmStorageContainerImmutableStorageWithVersioningMigration',
201+
'New-AzDataLakeGen2SasToken'
201202

202203
# Variables to export from this module
203204
# VariablesToExport = @()

‎src/Storage/Storage.Management/ChangeLog.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Supported generae Sas token for DataLakeGen2
22+
- `New-AzDataLakeGen2SasToken`
2123
* Show OAuth token in debug log in debug build only
2224
- `New-AzStorageContext`
2325

‎src/Storage/Storage.Management/help/Az.Storage.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,9 @@ Move a file or directory to another a file or directory in same Storage account.
197197
### [New-AzDataLakeGen2Item](New-AzDataLakeGen2Item.md)
198198
Create a file or directory in a filesystem.
199199

200+
### [New-AzDataLakeGen2SasToken](New-AzDataLakeGen2SasToken.md)
201+
Generates a SAS token for an Azure storage blob.
202+
200203
### [New-AzRmStorageContainer](New-AzRmStorageContainer.md)
201204
Creates a Storage blob container
202205

‎src/Storage/Storage.Management/help/New-AzDataLakeGen2SasToken.md‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
---
2+
external help file: Microsoft.Azure.PowerShell.Cmdlets.Storage.dll-Help.xml
3+
Module Name: Az.Storage
4+
online version: https://docs.microsoft.com/powershell/module/az.storage/new-azdatalakegen2sastoken
5+
schema: 2.0.0
6+
---
7+
8+
# New-AzDataLakeGen2SasToken
9+
10+
## SYNOPSIS
11+
Generates a SAS token for Azure DatalakeGen2 item.
12+
13+
## SYNTAX
14+
15+
### ReceiveManual (Default)
16+
```
17+
New-AzDataLakeGen2SasToken [-FileSystem] <String> [-Path <String>] [-Permission <String>]
18+
[-Protocol <SasProtocol>] [-IPAddressOrRange <String>] [-StartTime <DateTimeOffset>]
19+
[-ExpiryTime <DateTimeOffset>] [-FullUri] [-Context <IStorageContext>]
20+
[-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
21+
```
22+
23+
### ItemPipeline
24+
```
25+
New-AzDataLakeGen2SasToken -InputObject <AzureDataLakeGen2Item> [-Permission <String>]
26+
[-Protocol <SasProtocol>] [-IPAddressOrRange <String>] [-StartTime <DateTimeOffset>]
27+
[-ExpiryTime <DateTimeOffset>] [-FullUri] [-Context <IStorageContext>]
28+
[-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
29+
```
30+
31+
## DESCRIPTION
32+
The **New-AzDataLakeGen2SasToken** cmdlet generates a Shared Access Signature (SAS) token for an Azure DatalakeGen2 item.
33+
34+
## EXAMPLES
35+
36+
### Example 1: Generate a SAS token with full permission
37+
```
38+
New-AzDataLakeGen2SasToken -FileSystem "filesystem1" -Path "dir1/dir2" -Permission racwdlmeop
39+
```
40+
41+
This example generates a DatalakeGen2 SAS token with full permission.
42+
43+
### Example 2: Generate a SAS token with specific StartTime, ExpireTime, Protocal, IPAddressOrRange, by pipeline a datalakegen2 item
44+
```
45+
Get-AzDataLakeGen2Item -FileSystem test -Path "testdir/dir2" | New-AzDataLakeGen2SasToken -Permission rw -Protocol Https -IPAddressOrRange 10.0.0.0-12.10.0.0 -StartTime (Get-Date) -ExpiryTime (Get-Date).AddDays(6)
46+
```
47+
48+
This example generates a DatalakeGen2 SAS token by pipeline a datalake gen2 item, and with specific StartTime, ExpireTime, Protocal, IPAddressOrRange.
49+
50+
## PARAMETERS
51+
52+
### -Context
53+
Azure Storage Context Object
54+
55+
```yaml
56+
Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext
57+
Parameter Sets: (All)
58+
Aliases:
59+
60+
Required: False
61+
Position: Named
62+
Default value: None
63+
Accept pipeline input: True (ByPropertyName, ByValue)
64+
Accept wildcard characters: False
65+
```
66+
67+
### -DefaultProfile
68+
The credentials, account, tenant, and subscription used for communication with Azure.
69+
70+
```yaml
71+
Type: Microsoft.Azure.Commands.Common.Authentication.Abstractions.Core.IAzureContextContainer
72+
Parameter Sets: (All)
73+
Aliases: AzureRmContext, AzureCredential
74+
75+
Required: False
76+
Position: Named
77+
Default value: None
78+
Accept pipeline input: False
79+
Accept wildcard characters: False
80+
```
81+
82+
### -ExpiryTime
83+
Expiry Time
84+
85+
```yaml
86+
Type: System.Nullable`1[System.DateTimeOffset]
87+
Parameter Sets: (All)
88+
Aliases:
89+
90+
Required: False
91+
Position: Named
92+
Default value: None
93+
Accept pipeline input: False
94+
Accept wildcard characters: False
95+
```
96+
97+
### -FileSystem
98+
FileSystem name
99+
100+
```yaml
101+
Type: System.String
102+
Parameter Sets: ReceiveManual
103+
Aliases:
104+
105+
Required: True
106+
Position: 0
107+
Default value: None
108+
Accept pipeline input: True (ByValue)
109+
Accept wildcard characters: False
110+
```
111+
112+
### -FullUri
113+
Display full uri with sas token
114+
115+
```yaml
116+
Type: System.Management.Automation.SwitchParameter
117+
Parameter Sets: (All)
118+
Aliases:
119+
120+
Required: False
121+
Position: Named
122+
Default value: None
123+
Accept pipeline input: False
124+
Accept wildcard characters: False
125+
```
126+
127+
### -InputObject
128+
Azure Datalake Gen2 Item Object to remove.
129+
130+
```yaml
131+
Type: Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item
132+
Parameter Sets: ItemPipeline
133+
Aliases:
134+
135+
Required: True
136+
Position: Named
137+
Default value: None
138+
Accept pipeline input: True (ByValue)
139+
Accept wildcard characters: False
140+
```
141+
142+
### -IPAddressOrRange
143+
IP, or IP range ACL (access control list) that the request would be accepted by Azure Storage.
144+
145+
```yaml
146+
Type: System.String
147+
Parameter Sets: (All)
148+
Aliases:
149+
150+
Required: False
151+
Position: Named
152+
Default value: None
153+
Accept pipeline input: False
154+
Accept wildcard characters: False
155+
```
156+
157+
### -Path
158+
The path in the specified FileSystem that should be retrieved.
159+
Can be a file or directory In the format 'directory/file.txt' or 'directory1/directory2/'.
160+
Skip set this parameter to get the root directory of the Filesystem.
161+
162+
```yaml
163+
Type: System.String
164+
Parameter Sets: ReceiveManual
165+
Aliases:
166+
167+
Required: False
168+
Position: Named
169+
Default value: None
170+
Accept pipeline input: True (ByValue)
171+
Accept wildcard characters: False
172+
```
173+
174+
### -Permission
175+
Permissions for a blob.
176+
Permissions can be any not-empty subset of "racwdlmeop".
177+
178+
```yaml
179+
Type: System.String
180+
Parameter Sets: (All)
181+
Aliases:
182+
183+
Required: False
184+
Position: Named
185+
Default value: None
186+
Accept pipeline input: False
187+
Accept wildcard characters: False
188+
```
189+
190+
### -Protocol
191+
Protocol can be used in the request with this SAS token.
192+
193+
```yaml
194+
Type: System.Nullable`1[Azure.Storage.Sas.SasProtocol]
195+
Parameter Sets: (All)
196+
Aliases:
197+
Accepted values: None, HttpsAndHttp, Https
198+
199+
Required: False
200+
Position: Named
201+
Default value: None
202+
Accept pipeline input: False
203+
Accept wildcard characters: False
204+
```
205+
206+
### -StartTime
207+
Start Time
208+
209+
```yaml
210+
Type: System.Nullable`1[System.DateTimeOffset]
211+
Parameter Sets: (All)
212+
Aliases:
213+
214+
Required: False
215+
Position: Named
216+
Default value: None
217+
Accept pipeline input: False
218+
Accept wildcard characters: False
219+
```
220+
221+
### CommonParameters
222+
This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
223+
224+
## INPUTS
225+
226+
### System.String
227+
228+
### Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.AzureDataLakeGen2Item
229+
230+
### Microsoft.Azure.Commands.Common.Authentication.Abstractions.IStorageContext
231+
232+
## OUTPUTS
233+
234+
### System.String
235+
236+
## NOTES
237+
238+
## RELATED LINKS

‎src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,24 @@ public BlobRequestOptions RequestOptions
7474
}
7575
}
7676

77+
public DataLakeClientOptions DataLakeClientOptions
78+
{
79+
get
80+
{
81+
if (dataLakeClientOptions == null)
82+
{
83+
dataLakeClientOptions = new DataLakeClientOptions();
84+
dataLakeClientOptions.AddPolicy(new UserAgentPolicy(ApiConstants.UserAgentHeaderValue), HttpPipelinePosition.PerCall);
85+
return dataLakeClientOptions;
86+
}
87+
else
88+
{
89+
return dataLakeClientOptions;
90+
}
91+
}
92+
}
93+
private DataLakeClientOptions dataLakeClientOptions = null;
94+
7795
public BlobClientOptions ClientOptions
7896
{
7997
get
@@ -664,20 +682,20 @@ internal DataLakeFileSystemClient GetFileSystemClientByName(IStorageBlobManageme
664682

665683
if (localChannel.StorageContext.StorageAccount.Credentials.IsToken) //Oauth
666684
{
667-
fileSystem = new DataLakeFileSystemClient(fileSystemUri, localChannel.StorageContext.Track2OauthToken);
685+
fileSystem = new DataLakeFileSystemClient(fileSystemUri, localChannel.StorageContext.Track2OauthToken, this.DataLakeClientOptions);
668686
}
669687
else if (localChannel.StorageContext.StorageAccount.Credentials.IsSAS) //SAS
670688
{
671-
fileSystem = new DataLakeFileSystemClient(new Uri (fileSystemUri.ToString() + "?" + Util.GetSASStringWithoutQuestionMark(localChannel.StorageContext.StorageAccount.Credentials.SASToken)));
689+
fileSystem = new DataLakeFileSystemClient(new Uri (fileSystemUri.ToString() + "?" + Util.GetSASStringWithoutQuestionMark(localChannel.StorageContext.StorageAccount.Credentials.SASToken)), this.DataLakeClientOptions);
672690
}
673691
else if (localChannel.StorageContext.StorageAccount.Credentials.IsSharedKey) //Shared Key
674692
{
675693
fileSystem = new DataLakeFileSystemClient(fileSystemUri,
676-
new StorageSharedKeyCredential(localChannel.StorageContext.StorageAccountName, localChannel.StorageContext.StorageAccount.Credentials.ExportBase64EncodedKey()));
694+
new StorageSharedKeyCredential(localChannel.StorageContext.StorageAccountName, localChannel.StorageContext.StorageAccount.Credentials.ExportBase64EncodedKey()), this.DataLakeClientOptions);
677695
}
678696
else //Anonymous
679697
{
680-
fileSystem = new DataLakeFileSystemClient(fileSystemUri);
698+
fileSystem = new DataLakeFileSystemClient(fileSystemUri, this.DataLakeClientOptions);
681699
}
682700

683701
return fileSystem;

‎src/Storage/Storage/Common/AzureDataLakeGen2Item.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@ public class AzureDataLakeGen2Item : AzureStorageBase
103103
[Ps1Xml(Label = "Group", Target = ViewControl.Table, Position = 7, TableColumnWidth = 10)]
104104
public string Group { get; set; }
105105

106+
/// <summary>
107+
/// The PathItem properties of the item, the property only exist if the item is listout
108+
/// </summary>
109+
public PathItem ListPathItem { get; set; }
110+
106111
/// <summary>
107112
/// Azure DataLakeGen2 Item constructor
108113
/// </summary>
@@ -190,6 +195,7 @@ public AzureDataLakeGen2Item(PathItem item, DataLakeFileSystemClient fileSystem,
190195
{
191196
this.Name = item.Name;
192197
this.Path = item.Name;
198+
this.ListPathItem = item;
193199
this.IsDirectory = item.IsDirectory is null ? false : item.IsDirectory.Value;
194200
DataLakePathClient pathclient = null;
195201
if (this.IsDirectory) // Directory

0 commit comments

Comments
 (0)