Skip to content

Commit 94c2c5a

Browse files
yifanz0yifanz0
authored
[Storage] Support excludePrefix, includeDeleted, and new schema fields in blob inventory (#20068)
* support excludePrefix, includeDeleted, and new schema fields in blob inventory * fix per comments * fix
1 parent 8010af8 commit 94c2c5a

File tree

8 files changed

+1120
-877
lines changed

8 files changed

+1120
-877
lines changed

‎src/Storage/Storage.Management.Test/ScenarioTests/StorageAccountTests.ps1‎

Lines changed: 501 additions & 445 deletions
Large diffs are not rendered by default.

‎src/Storage/Storage.Management.Test/SessionRecords/Microsoft.Azure.Commands.Management.Storage.Test.ScenarioTests.StorageAccountTests/TestStorageBlobInventory.json‎

Lines changed: 393 additions & 393 deletions
Large diffs are not rendered by default.

‎src/Storage/Storage.Management/ChangeLog.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Supported excludePrefix, includeDeleted, and many new schema fields in Blob Inventory
22+
- `New-AzStorageBlobInventoryPolicyRule`
2123

2224
## Version 5.1.0
2325
* Supported generate DataLakeGen2 Sas token with Encryption scope

‎src/Storage/Storage.Management/Models/PSBlobInventoryPolicy.cs‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -179,11 +179,16 @@ public BlobInventoryPolicyDefinition parseBlobInventoryPolicyDefinition()
179179
// Possible values include: 'Blob', 'Container'
180180
public string ObjectType { get; set; }
181181

182-
// Valid values for this field for the blob object type include: Name, Creation-Time, Last-Modified, Content-Length,
183-
// Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder,
184-
// Owner, Group, Permissions, Acl, Snapshot, VersionId, IsCurrentVersion, Metadata, LastAccessTime, AccessTierInferred, Tags.
185-
// Valid values for container object type include Name, Last-Modified,
186-
// Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold.
182+
// Valid values for this field for the blob object type include: Name, Creation-Time, Last-Modified, Content-Length,
183+
// Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder,
184+
// Owner, Group, Permissions, Acl, Snapshot, VersionId, IsCurrentVersion, Metadata, LastAccessTime, AccessTierInferred, Tags,
185+
// Etag, Content-Type, Content-Encoding, Content-Language, Content-CRC64, Cache-Control, Content-Disposition,
186+
// LeaseStatus, LeaseState, LeaseDuration, ServerEncrypted, Deleted, DeletionId, DeletedTime, RemainingRetentionDays,
187+
// mmutabilityPolicyUntilDate, ImmutabilityPolicyMode, LegalHold, CopyId, CopyStatus, CopySource, CopyProgress, CopyCompletionTime,
188+
// CopyStatusDescription, CustomerProvidedKeySha256, RehydratePriority, ArchiveStatus, x-ms-blob-sequence-number, EncryptionScope, IncrementalCopy, TagCount
189+
// Valid values for container object type include Name, Last-Modified,
190+
// Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold, Etag,
191+
// DefaultEncryptionScope, DenyEncryptionScopeOverride, ImmutableStorageWithVersioningEnabled, Deleted, Version, DeletedTime, RemainingRetentionDays
187192
public string[] SchemaFields { get; set; }
188193

189194
//private string[] BlobSchemaField = new string[] {"Name", "Creation-Time", "Last-Modified", "Content-Length", "Content-MD5", "BlobType", "AccessTier", "AccessTierChangeTime",
@@ -216,26 +221,32 @@ public PSBlobInventoryPolicyFilter() { }
216221
public PSBlobInventoryPolicyFilter(BlobInventoryPolicyFilter filters)
217222
{
218223
this.PrefixMatch = PSManagementPolicyRuleFilter.StringListToArray(filters.PrefixMatch);
224+
this.ExcludePrefix = PSManagementPolicyRuleFilter.StringListToArray(filters.ExcludePrefix);
219225
this.BlobTypes = PSManagementPolicyRuleFilter.StringListToArray(filters.BlobTypes);
220226
this.IncludeBlobVersions = filters.IncludeBlobVersions;
221227
this.IncludeSnapshots = filters.IncludeSnapshots;
228+
this.IncludeDeleted = filters.IncludeDeleted;
222229
}
223230

224231
public BlobInventoryPolicyFilter ParseBlobInventoryPolicyFilter()
225232
{
226233
return new BlobInventoryPolicyFilter()
227234
{
228235
PrefixMatch = PSManagementPolicyRuleFilter.StringArrayToList(this.PrefixMatch),
236+
ExcludePrefix = PSManagementPolicyRuleFilter.StringArrayToList(this.ExcludePrefix),
229237
BlobTypes = PSManagementPolicyRuleFilter.StringArrayToList(this.BlobTypes),
230238
IncludeSnapshots = this.IncludeSnapshots,
231-
IncludeBlobVersions = this.IncludeBlobVersions
239+
IncludeBlobVersions = this.IncludeBlobVersions,
240+
IncludeDeleted = this.IncludeDeleted
232241
};
233242
}
234243

235244
public string[] PrefixMatch { get; set; }
245+
public string[] ExcludePrefix { get; set; }
236246
public string[] BlobTypes { get; set; }
237247
public bool? IncludeBlobVersions { get; set; }
238248
public bool? IncludeSnapshots { get; set; }
249+
public bool? IncludeDeleted { get; set; }
239250
}
240251

241252
/// <summary>

‎src/Storage/Storage.Management/Storage.Management.format.ps1xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,10 @@
947947
<Alignment>Left</Alignment>
948948
<Label>IncludeBlobVersions</Label>
949949
</TableColumnHeader>
950+
<TableColumnHeader>
951+
<Alignment>Left</Alignment>
952+
<Label>IncludeDeleted</Label>
953+
</TableColumnHeader>
950954
<TableColumnHeader>
951955
<Alignment>Left</Alignment>
952956
<Label>BlobTypes</Label>
@@ -955,6 +959,10 @@
955959
<Alignment>Left</Alignment>
956960
<Label>PrefixMatch</Label>
957961
</TableColumnHeader>
962+
<TableColumnHeader>
963+
<Alignment>Left</Alignment>
964+
<Label>ExcludePrefix</Label>
965+
</TableColumnHeader>
958966
<TableColumnHeader>
959967
<Alignment>Left</Alignment>
960968
<Label>SchemaFields</Label>
@@ -995,6 +1003,10 @@
9951003
<Alignment>Left</Alignment>
9961004
<ScriptBlock>$_.Definition.Filters.IncludeBlobVersions</ScriptBlock>
9971005
</TableColumnItem>
1006+
<TableColumnItem>
1007+
<Alignment>Left</Alignment>
1008+
<ScriptBlock>$_.Definition.Filters.IncludeDeleted</ScriptBlock>
1009+
</TableColumnItem>
9981010
<TableColumnItem>
9991011
<Alignment>Left</Alignment>
10001012
<ScriptBlock>$_.Definition.Filters.BlobTypes</ScriptBlock>
@@ -1003,6 +1015,10 @@
10031015
<Alignment>Left</Alignment>
10041016
<ScriptBlock>$_.Definition.Filters.PrefixMatch</ScriptBlock>
10051017
</TableColumnItem>
1018+
<TableColumnItem>
1019+
<Alignment>Left</Alignment>
1020+
<ScriptBlock>$_.Definition.Filters.ExcludePrefix</ScriptBlock>
1021+
</TableColumnItem>
10061022
<TableColumnItem>
10071023
<Alignment>Left</Alignment>
10081024
<ScriptBlock>$_.Definition.SchemaFields</ScriptBlock>

‎src/Storage/Storage.Management/StorageAccount/NewAzureStorageBlobInventoryPolicyRule.cs‎

Lines changed: 107 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,13 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
6666
ParameterSetName = BlobRuleParameterSet,
6767
HelpMessage = "Specifies the fields and properties of the Blob object to be included in the inventory. Valid values include: " +
6868
"Name, Creation-Time, Last-Modified, Content-Length, Content-MD5, BlobType, AccessTier, AccessTierChangeTime, Expiry-Time, hdi_isfolder, Owner, " +
69-
"Group, Permissions, Acl, Metadata, LastAccessTime, AccessTierInferred, Tags. 'Name' is a required schemafield. " +
69+
"Group, Permissions, Acl, Metadata, LastAccessTime, AccessTierInferred, Tags, Etag, Content-Type, Content-Encoding, Content-Language, Content-CRC64, " +
70+
"Cache-Control, Content-Disposition, LeaseStatus, LeaseState, LeaseDuration, ServerEncrypted, Deleted, RemainingRetentionDays, ImmutabilityPolicyUntilDate" +
71+
"ImmutabilityPolicyMode, LegalHold, CopyId, CopyStatus, CopySource, CopyProgress, CopyCompletionTime, CopyStatusDescription, CustomerProvidedKeySha256 " +
72+
"RehydratePriority, ArchiveStatus, x-ms-blob-sequence-number, EncryptionScope, IncrementalCopy, DeletionId, DeletedTime, TagCount. " +
73+
"'Name' is a required schemafield. " +
7074
"Schema field values 'Expiry-Time, hdi_isfolder, Owner, Group, Permissions, Acl' are valid only for Hns enabled accounts.'Tags' field is only valid for non Hns accounts." +
75+
"'Tags, TagCount' field is only valid for non Hns accounts. " +
7176
"If specify '-IncludeSnapshot', will include 'Snapshot' in the inventory. If specify '-IncludeBlobVersion', will include 'VersionId, 'IsCurrentVersion' in the inventory.")]
7277
[ValidateSet(BlobInventoryPolicyBlobSchemaField.Name,
7378
BlobInventoryPolicyBlobSchemaField.CreationTime,
@@ -87,15 +92,47 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
8792
BlobInventoryPolicyBlobSchemaField.LastAccessTime,
8893
BlobInventoryPolicyBlobSchemaField.AccessTierInferred,
8994
BlobInventoryPolicyBlobSchemaField.Tags,
95+
BlobInventoryPolicyBlobSchemaField.Etag,
96+
BlobInventoryPolicyBlobSchemaField.ContentType,
97+
BlobInventoryPolicyBlobSchemaField.ContentEncoding,
98+
BlobInventoryPolicyBlobSchemaField.ContentLanguage,
99+
BlobInventoryPolicyBlobSchemaField.ContentCRC64,
100+
BlobInventoryPolicyBlobSchemaField.CacheControl,
101+
BlobInventoryPolicyBlobSchemaField.ContentDisposition,
102+
BlobInventoryPolicyBlobSchemaField.LeaseStatus,
103+
BlobInventoryPolicyBlobSchemaField.LeaseState,
104+
BlobInventoryPolicyBlobSchemaField.LeaseDuration,
105+
BlobInventoryPolicyBlobSchemaField.ServerEncrypted,
106+
BlobInventoryPolicyBlobSchemaField.Deleted,
107+
BlobInventoryPolicyBlobSchemaField.RemainingRetentionDays,
108+
BlobInventoryPolicyBlobSchemaField.ImmutabilityPolicyUntilDate,
109+
BlobInventoryPolicyBlobSchemaField.ImmutabilityPolicyMode,
110+
BlobInventoryPolicyBlobSchemaField.LegalHold,
111+
BlobInventoryPolicyBlobSchemaField.CopyId,
112+
BlobInventoryPolicyBlobSchemaField.CopyStatus,
113+
BlobInventoryPolicyBlobSchemaField.CopySource,
114+
BlobInventoryPolicyBlobSchemaField.CopyProgress,
115+
BlobInventoryPolicyBlobSchemaField.CopyCompletionTime,
116+
BlobInventoryPolicyBlobSchemaField.CopyStatusDescription,
117+
BlobInventoryPolicyBlobSchemaField.CustomerProvidedKeySha256,
118+
BlobInventoryPolicyBlobSchemaField.RehydratePriority,
119+
BlobInventoryPolicyBlobSchemaField.ArchiveStatus,
120+
BlobInventoryPolicyBlobSchemaField.xmsblobsequencenumber,
121+
BlobInventoryPolicyBlobSchemaField.EncryptionScope,
122+
BlobInventoryPolicyBlobSchemaField.IncrementalCopy,
123+
BlobInventoryPolicyBlobSchemaField.DeletionId,
124+
BlobInventoryPolicyBlobSchemaField.DeletedTime,
125+
BlobInventoryPolicyBlobSchemaField.TagCount,
90126
IgnoreCase = true)]
91127
[ValidateNotNullOrEmpty]
92128
public string[] BlobSchemaField { get; set; }
93129

94130
[Parameter(Mandatory = true,
95131
ParameterSetName = ContainerRuleParameterSet,
96132
HelpMessage = "Specifies the fields and properties of the container object to be included in the inventory. Valid values include: " +
97-
"Name, Last-Modified, Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold. " +
98-
"'Name' is a required schemafield.")]
133+
"Name, Last-Modified, Metadata, LeaseStatus, LeaseState, LeaseDuration, PublicAccess, HasImmutabilityPolicy, HasLegalHold, " +
134+
"Etag, DefaultEncryptionScope, DenyEncryptionScopeOverride, ImmutableStorageWithVersioningEnabled, Deleted, Version, " +
135+
"DeletedTime, RemainingRetentionDays. 'Name' is a required schemafield.")]
99136
[ValidateSet(BlobInventoryPolicyContainerSchemaField.Name,
100137
BlobInventoryPolicyContainerSchemaField.LastModified,
101138
BlobInventoryPolicyContainerSchemaField.Metadata,
@@ -105,6 +142,14 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
105142
BlobInventoryPolicyContainerSchemaField.PublicAccess,
106143
BlobInventoryPolicyContainerSchemaField.HasImmutabilityPolicy,
107144
BlobInventoryPolicyContainerSchemaField.HasLegalHold,
145+
BlobInventoryPolicyContainerSchemaField.Etag,
146+
BlobInventoryPolicyContainerSchemaField.DefaultEncryptionScope,
147+
BlobInventoryPolicyContainerSchemaField.DenyEncryptionScopeOverride,
148+
BlobInventoryPolicyContainerSchemaField.ImmutableStorageWithVersioningEnabled,
149+
BlobInventoryPolicyContainerSchemaField.Deleted,
150+
BlobInventoryPolicyContainerSchemaField.Version,
151+
BlobInventoryPolicyContainerSchemaField.DeletedTime,
152+
BlobInventoryPolicyContainerSchemaField.RemainingRetentionDays,
108153
IgnoreCase = true)]
109154
[ValidateNotNullOrEmpty]
110155
public string[] ContainerSchemaField { get; set; }
@@ -120,31 +165,42 @@ public class NewAzureStorageBlobInventoryPolicyRuleCommand : StorageAccountBaseC
120165
public string[] BlobType { get; set; }
121166

122167
[Parameter(Mandatory = false,
123-
HelpMessage = "Sets an array of strings for blob prefixes to be matched..")]
168+
HelpMessage = "Sets an array of strings for blob prefixes to be matched.")]
124169
[ValidateNotNullOrEmpty]
125170
public string[] PrefixMatch { get; set; }
126171

172+
[Parameter(Mandatory = false,
173+
HelpMessage = "Sets an array of strings with maximum 10 blob prefixes to be excluded from the inventory.")]
174+
[ValidateNotNullOrEmpty]
175+
public string[] ExcludePrefix { get; set; }
176+
127177
[Parameter(Mandatory = false,
128178
ParameterSetName = BlobRuleParameterSet,
129-
HelpMessage = "The rule is disabled if set it.")]
179+
HelpMessage = "Includes blob snapshots in blob inventory")]
130180
public SwitchParameter IncludeSnapshot { get; set; }
131181

132182
[Parameter(Mandatory = false,
133183
ParameterSetName = BlobRuleParameterSet,
134-
HelpMessage = "The rule is disabled if set it.")]
184+
HelpMessage = "Includes blob versions in blob inventory.")]
135185
public SwitchParameter IncludeBlobVersion { get; set; }
136186

187+
[Parameter(Mandatory = false,
188+
ParameterSetName = BlobRuleParameterSet,
189+
HelpMessage = "Includes deleted blob in blob inventory. When include delete blob, for ContainerSchemaFields, must include 'Deleted, Version, DeletedTime and RemainingRetentionDays'. For BlobSchemaFields, on HNS enabled storage accounts, must include 'DeletionId, Deleted, DeletedTime and RemainingRetentionDays', and on Hns disabled accounts must include 'Deleted and RemainingRetentionDays', else they must be excluded.")]
190+
public SwitchParameter IncludeDeleted { get; set; }
191+
137192
public override void ExecuteCmdlet()
138193
{
139194
base.ExecuteCmdlet();
140195

141196
PSBlobInventoryPolicyDefinition definition = new PSBlobInventoryPolicyDefinition();
142-
if (this.BlobType != null || this.PrefixMatch != null || this.IncludeSnapshot.IsPresent || this.IncludeBlobVersion.IsPresent)
197+
if (this.BlobType != null || this.PrefixMatch != null || this.ExcludePrefix != null || this.IncludeSnapshot.IsPresent || this.IncludeBlobVersion.IsPresent || this.IncludeDeleted.IsPresent)
143198
{
144199
definition.Filters = new PSBlobInventoryPolicyFilter()
145200
{
146201
BlobTypes = NormalizeStringArray<AzureBlobType>(this.BlobType),
147-
PrefixMatch = this.PrefixMatch
202+
PrefixMatch = this.PrefixMatch,
203+
ExcludePrefix = this.ExcludePrefix
148204
};
149205
if (this.IncludeBlobVersion.IsPresent)
150206
{
@@ -154,6 +210,10 @@ public override void ExecuteCmdlet()
154210
{
155211
definition.Filters.IncludeSnapshots = true;
156212
}
213+
if (this.IncludeDeleted.IsPresent)
214+
{
215+
definition.Filters.IncludeDeleted = true;
216+
}
157217
}
158218
definition.Format = NormalizeString<BlobInventoryPolicyRuleFormat>(this.Format);
159219
definition.Schedule = NormalizeString<BlobInventoryPolicyRuleSchedule>(this.Schedule);
@@ -218,6 +278,14 @@ protected struct BlobInventoryPolicyContainerSchemaField
218278
public const string PublicAccess = "PublicAccess";
219279
public const string HasImmutabilityPolicy = "HasImmutabilityPolicy";
220280
public const string HasLegalHold = "HasLegalHold";
281+
public const string Etag = "Etag";
282+
public const string DefaultEncryptionScope = "DefaultEncryptionScope";
283+
public const string DenyEncryptionScopeOverride = "DenyEncryptionScopeOverride";
284+
public const string ImmutableStorageWithVersioningEnabled = "ImmutableStorageWithVersioningEnabled";
285+
public const string Deleted = "Deleted";
286+
public const string Version = "Version";
287+
public const string DeletedTime = "DeletedTime";
288+
public const string RemainingRetentionDays = "RemainingRetentionDays";
221289
}
222290

223291
protected struct BlobInventoryPolicyBlobSchemaField
@@ -243,6 +311,37 @@ protected struct BlobInventoryPolicyBlobSchemaField
243311
public const string LastAccessTime = "LastAccessTime";
244312
public const string AccessTierInferred = "AccessTierInferred";
245313
public const string Tags = "Tags";
314+
public const string Etag = "Etag";
315+
public const string ContentType = "Content-Type";
316+
public const string ContentEncoding = "Content-Encoding";
317+
public const string ContentLanguage = "Content-Language";
318+
public const string ContentCRC64 = "Content-CRC64";
319+
public const string CacheControl = "Cache-Control";
320+
public const string ContentDisposition = "Content-Disposition";
321+
public const string LeaseStatus = "LeaseStatus";
322+
public const string LeaseState = "LeaseState";
323+
public const string LeaseDuration = "LeaseDuration";
324+
public const string ServerEncrypted = "ServerEncrypted";
325+
public const string Deleted = "Deleted";
326+
public const string RemainingRetentionDays = "RemainingRetentionDays";
327+
public const string ImmutabilityPolicyUntilDate = "ImmutabilityPolicyUntilDate";
328+
public const string ImmutabilityPolicyMode = "ImmutabilityPolicyMode";
329+
public const string LegalHold = "LegalHold";
330+
public const string CopyId = "CopyId";
331+
public const string CopyStatus = "CopyStatus";
332+
public const string CopySource = "CopySource";
333+
public const string CopyProgress = "CopyProgress";
334+
public const string CopyCompletionTime = "CopyCompletionTime";
335+
public const string CopyStatusDescription = "CopyStatusDescription";
336+
public const string CustomerProvidedKeySha256 = "CustomerProvidedKeySha256";
337+
public const string RehydratePriority = "RehydratePriority";
338+
public const string ArchiveStatus = "ArchiveStatus";
339+
public const string xmsblobsequencenumber = "x-ms-blob-sequence-number";
340+
public const string EncryptionScope = "EncryptionScope";
341+
public const string IncrementalCopy = "IncrementalCopy";
342+
public const string DeletionId = "DeletionId";
343+
public const string DeletedTime = "DeletedTime";
344+
public const string TagCount = "TagCount";
246345
}
247346

248347
protected struct BlobInventoryPolicyRuleSchedule

0 commit comments

Comments
 (0)