diff --git a/src/Accounts/AssemblyLoading/ConditionalAssemblyProvider.cs b/src/Accounts/AssemblyLoading/ConditionalAssemblyProvider.cs index 1740e03a7d28..a8a52de4304d 100644 --- a/src/Accounts/AssemblyLoading/ConditionalAssemblyProvider.cs +++ b/src/Accounts/AssemblyLoading/ConditionalAssemblyProvider.cs @@ -51,7 +51,7 @@ public static void Initialize(string rootPath, IConditionalAssemblyContext conte CreateAssembly("net462", "System.Diagnostics.DiagnosticSource", "8.0.0.1").WithWindowsPowerShell(), CreateAssembly("net462", "System.Text.Encodings.Web", "8.0.0.0").WithWindowsPowerShell(), CreateAssembly("net47", "System.Security.Cryptography.Cng", "4.3.0.0").WithWindowsPowerShell(), - CreateAssembly("netstandard2.0", "Azure.Core", "1.47.3.0"), + CreateAssembly("netstandard2.0", "Azure.Core", "1.50.0.0"), CreateAssembly("netstandard2.0", "Azure.Identity.Broker", "1.1.0.0"), CreateAssembly("netstandard2.0", "Azure.Identity", "1.13.0.0"), CreateAssembly("netstandard2.0", "Microsoft.Bcl.AsyncInterfaces", "8.0.0.0"), @@ -61,7 +61,7 @@ public static void Initialize(string rootPath, IConditionalAssemblyContext conte CreateAssembly("netstandard2.0", "Microsoft.Identity.Client.NativeInterop", "0.16.2.0"), CreateAssembly("netstandard2.0", "Microsoft.IdentityModel.Abstractions", "6.35.0.0"), CreateAssembly("netstandard2.0", "System.Buffers", "4.0.3.0").WithWindowsPowerShell(), - CreateAssembly("netstandard2.0", "System.ClientModel", "1.6.1.0"), + CreateAssembly("netstandard2.0", "System.ClientModel", "1.8.0.0"), CreateAssembly("netstandard2.0", "System.Memory.Data", "8.0.0.1"), CreateAssembly("netstandard2.0", "System.Memory", "4.0.1.2").WithWindowsPowerShell(), CreateAssembly("netstandard2.0", "System.Net.Http.WinHttpHandler", "4.0.4.0").WithWindowsPowerShell(), diff --git a/src/Storage/Storage.Test/Blob/StorageCloudBlobCmdletBaseTest.cs b/src/Storage/Storage.Test/Blob/StorageCloudBlobCmdletBaseTest.cs index 2cedf4ea7f9e..32721add73b2 100644 --- a/src/Storage/Storage.Test/Blob/StorageCloudBlobCmdletBaseTest.cs +++ b/src/Storage/Storage.Test/Blob/StorageCloudBlobCmdletBaseTest.cs @@ -33,7 +33,7 @@ public void InitCommand() { command = new StorageCloudBlobCmdletBase(BlobMock) { - Context = new AzureStorageContext(CloudStorageAccount.DevelopmentStorageAccount), + Context = new AzureStorageContext(CloudStorageAccount.DevelopmentStorageAccount, null, null, null), CommandRuntime = MockCmdRunTime }; } diff --git a/src/Storage/Storage.Test/Common/Cmdlet/NewAzureStorageContextTest.cs b/src/Storage/Storage.Test/Common/Cmdlet/NewAzureStorageContextTest.cs index e4bb308bf2bd..90a6802f3f87 100644 --- a/src/Storage/Storage.Test/Common/Cmdlet/NewAzureStorageContextTest.cs +++ b/src/Storage/Storage.Test/Common/Cmdlet/NewAzureStorageContextTest.cs @@ -114,10 +114,10 @@ public void GetStorageAccountByConnectionStringAndSasToken() string endpoint = "http://storageaccountname.blob.core.windows.net"; string connectionString = String.Format("BlobEndpoint={0};QueueEndpoint={0};TableEndpoint={0};SharedAccessSignature={1}", endpoint, sasToken); CloudStorageAccount account = command.GetStorageAccountByConnectionString(connectionString); - AzureStorageContext context = new AzureStorageContext(account); + AzureStorageContext context = new AzureStorageContext(account, null, null, null); connectionString = String.Format("BlobEndpoint={0};SharedAccessSignature={1}", endpoint, sasToken); account = command.GetStorageAccountByConnectionString(connectionString); - context = new AzureStorageContext(account); + context = new AzureStorageContext(account, null, null, null); } } } diff --git a/src/Storage/Storage.Test/Common/StorageCloudCmdletBaseTest.cs b/src/Storage/Storage.Test/Common/StorageCloudCmdletBaseTest.cs index 36a909494da6..581f6f00025c 100644 --- a/src/Storage/Storage.Test/Common/StorageCloudCmdletBaseTest.cs +++ b/src/Storage/Storage.Test/Common/StorageCloudCmdletBaseTest.cs @@ -54,7 +54,7 @@ public void CleanCommand() public void GetCloudStorageAccountFromContextTest() { CloudStorageAccount account = CloudStorageAccount.DevelopmentStorageAccount; - command.Context = new AzureStorageContext(account); + command.Context = new AzureStorageContext(account, null, null, null); Assert.AreEqual(command.Context, command.GetCmdletStorageContext()); } @@ -73,7 +73,7 @@ public void WriteObjectWithStorageContextWithNullContextTest() public void WriteObjectWithStorageContextWithContextTest() { CloudStorageAccount account = CloudStorageAccount.DevelopmentStorageAccount; - command.Context = new AzureStorageContext(account); + command.Context = new AzureStorageContext(account, null, null, null); AzureStorageBase item = new AzureStorageBase(); command.WriteObjectWithStorageContext(item); @@ -105,7 +105,7 @@ public void WriteObjectWithStorageContextWihtEnumerableList() public void ShouldInitServiceChannelTest() { CloudStorageAccount account = CloudStorageAccount.DevelopmentStorageAccount; - command.Context = new AzureStorageContext(account); + command.Context = new AzureStorageContext(account, null, null, null); string toss; Assert.IsFalse(command.TryGetStorageAccount(command.SMProfile, out toss)); } diff --git a/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs b/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs index a6a25232d605..23b1831be804 100644 --- a/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs +++ b/src/Storage/Storage.Test/Service/MockStorageBlobManagement.cs @@ -744,6 +744,11 @@ public BlobServiceClient GetBlobServiceClient(BlobClientOptions options = null) throw new NotImplementedException(); } + public bool IsSasWithOAuthCredential() + { + throw new NotImplementedException(); + } + /// /// The storage context /// diff --git a/src/Storage/Storage.common/Common/AzureContextAdapterExtensions.cs b/src/Storage/Storage.common/Common/AzureContextAdapterExtensions.cs index 2724ede3b406..4fd0a190df96 100644 --- a/src/Storage/Storage.common/Common/AzureContextAdapterExtensions.cs +++ b/src/Storage/Storage.common/Common/AzureContextAdapterExtensions.cs @@ -84,7 +84,9 @@ public static IStorageContext GetStorageContext(this IStorageService service) { return new AzureStorageContext(new CloudStorageAccount(new StorageCredentials(service.Name, service.AuthenticationKeys.First()), new StorageUri(service.BlobEndpoint), new StorageUri(service.QueueEndpoint), - new StorageUri(service.TableEndpoint), new StorageUri(service.FileEndpoint))); + new StorageUri(service.TableEndpoint), new StorageUri(service.FileEndpoint)), + null, + false); } diff --git a/src/Storage/Storage.common/Common/AzureStorageContext.cs b/src/Storage/Storage.common/Common/AzureStorageContext.cs index ebe02864c1cb..d3bcd17b3bf6 100644 --- a/src/Storage/Storage.common/Common/AzureStorageContext.cs +++ b/src/Storage/Storage.common/Common/AzureStorageContext.cs @@ -149,7 +149,20 @@ public string ConnectionString { /// Storage account name /// /// - public AzureStorageContext(CloudStorageAccount account, string accountName = null, IAzureContext DefaultContext = null, DebugLogWriter logWriter = null) + public AzureStorageContext(CloudStorageAccount account, string accountName = null, IAzureContext DefaultContext = null, DebugLogWriter logWriter = null) : + this(account, accountName, false, null, null) + { + } + + /// + /// Create a storage context usign cloud storage account + /// + /// cloud storage account + /// Storage account name + /// + /// + /// + public AzureStorageContext(CloudStorageAccount account, string accountName = null, bool isOAuthToken = false, IAzureContext DefaultContext = null, DebugLogWriter logWriter = null) { StorageAccount = account; TableStorageAccount = XTable.CloudStorageAccount.Parse(StorageAccount.ToString(true)); @@ -195,7 +208,8 @@ public AzureStorageContext(CloudStorageAccount account, string accountName = nul StorageAccountName = "[Anonymous]"; } } - if (account.Credentials != null && account.Credentials.IsToken) + if ((account.Credentials != null && account.Credentials.IsToken) + || isOAuthToken) { Track2OauthToken = new AzureSessionCredential(DefaultContext, logWriter); } diff --git a/src/Storage/Storage.common/Storage.common.csproj b/src/Storage/Storage.common/Storage.common.csproj index 4add4bf55085..21b8f849c5cd 100644 --- a/src/Storage/Storage.common/Storage.common.csproj +++ b/src/Storage/Storage.common/Storage.common.csproj @@ -17,10 +17,14 @@ - + + + + + \ No newline at end of file diff --git a/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs b/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs index 80950a78ad89..8f7a2865b112 100644 --- a/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs +++ b/src/Storage/Storage/Blob/Cmdlet/CopyAzureStorageBlob.cs @@ -15,8 +15,15 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet { + using System; + using System.Collections.Generic; + using System.Management.Automation; + using System.Security.Permissions; + using System.Threading.Tasks; using Azure.Commands.Common.Authentication.Abstractions; using Commands.Common.Storage.ResourceModel; + using global::Azure; + using global::Azure.Core; using global::Azure.Storage.Blobs; using global::Azure.Storage.Blobs.Models; using global::Azure.Storage.Blobs.Specialized; @@ -24,11 +31,6 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Blob.Cmdlet using Microsoft.Azure.Storage.Blob; using Microsoft.WindowsAzure.Commands.Storage.Common; using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; - using System; - using System.Collections.Generic; - using System.Management.Automation; - using System.Security.Permissions; - using System.Threading.Tasks; using Track2Models = global::Azure.Storage.Blobs.Models; [Cmdlet("Copy", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageBlob", SupportsShouldProcess = true, DefaultParameterSetName = ContainerNameParameterSet),OutputType(typeof(AzureStorageBlob))] @@ -323,7 +325,7 @@ private void CopyBlobSync(IStorageBlobManagement destChannel, BlobBaseClient src destCloudBlob = Util.GetTrack2BlobClientWithType(destCloudBlob, destChannel.StorageContext, srcBlobType, ClientOptions); } - Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, srcCloudBlob.GenerateUriWithCredentials(Channel.StorageContext), destCloudBlob); + Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, srcCloudBlob.GenerateUriWithCredentials(Channel.StorageContext), Channel, destCloudBlob); RunTask(taskGenerator); } @@ -332,15 +334,24 @@ private void CopyBlobSync(IStorageBlobManagement destChannel, string srcUri, str Track2Models.BlobType srcBlobType = Util.GetBlobType(new BlobBaseClient(new Uri(srcUri), ClientOptions), true).Value; BlobBaseClient destBlob = this.GetDestBlob(destChannel, destContainer, destBlobName, srcBlobType); - Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, new Uri(srcUri), destBlob); + Func taskGenerator = (taskId) => CopyFromUri(taskId, destChannel, new Uri(srcUri), Channel, destBlob); RunTask(taskGenerator); } - private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, Uri srcUri, BlobBaseClient destBlob) + private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, Uri srcUri, IStorageBlobManagement sourceChannel, BlobBaseClient destBlob) { bool destExist = true; - Track2Models.BlobType? srcBlobType = Util.GetBlobType(new BlobBaseClient(srcUri, ClientOptions), true).Value; - Track2Models.BlobType? destBlobType = Util.GetBlobType(new BlobBaseClient(srcUri, ClientOptions), true).Value; + BlobBaseClient srcBlobClient; + if (sourceChannel.StorageContext != null && sourceChannel.StorageContext.Track2OauthToken != null) + { + srcBlobClient = new BlobBaseClient(srcUri, sourceChannel.StorageContext.Track2OauthToken, ClientOptions); + } + else + { + srcBlobClient = new BlobBaseClient(srcUri, ClientOptions); + } + Track2Models.BlobType? srcBlobType = Util.GetBlobType(srcBlobClient, true).Value; + Track2Models.BlobType? destBlobType = srcBlobType; Track2Models.BlobProperties properties = null; try @@ -398,8 +409,6 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, if (!destExist || this.ConfirmOverwrite(srcUri.AbsoluteUri.ToString(), destBlob.Uri.ToString())) { - - BlobBaseClient srcBlobClient= new BlobBaseClient(srcUri, ClientOptions); Track2Models.BlobProperties srcProperties = srcBlobClient.GetProperties(cancellationToken: this.CmdletCancellationToken).Value; Track2Models.BlobHttpHeaders httpHeaders = new Track2Models.BlobHttpHeaders @@ -453,6 +462,12 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, destPageBlob.Create(srcProperties.ContentLength, pageBlobCreateOptions, this.CmdletCancellationToken); Track2Models.PageBlobUploadPagesFromUriOptions pageBlobUploadPagesFromUriOptions = new Track2Models.PageBlobUploadPagesFromUriOptions(); + if (sourceChannel.StorageContext != null && sourceChannel.StorageContext.Track2OauthToken != null) + { + string oauthToken = sourceChannel.StorageContext.Track2OauthToken.GetToken(null, this.CmdletCancellationToken).TokenValue; + pageBlobUploadPagesFromUriOptions.SourceAuthentication = new HttpAuthorization("Bearer", oauthToken); + } + long pageCopyOffset = 0; progressHandler.Report(pageCopyOffset); long contentLenLeft = srcProperties.ContentLength; @@ -488,6 +503,12 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, SourceRange = new global::Azure.HttpRange(appendCopyOffset, appendContentSize) }; + if (sourceChannel.StorageContext != null && sourceChannel.StorageContext.Track2OauthToken != null) + { + string oauthToken = sourceChannel.StorageContext.Track2OauthToken.GetToken(null, this.CmdletCancellationToken).TokenValue; + appendBlobAppendBlockFromUriOptions.SourceAuthentication = new HttpAuthorization("Bearer", oauthToken); + } + destAppendBlob.AppendBlockFromUri(srcUri, appendBlobAppendBlockFromUriOptions, this.CmdletCancellationToken); appendCopyOffset += appendContentSize; progressHandler.Report(appendContentSize); @@ -513,6 +534,12 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, options.Metadata = srcProperties.Metadata; options.Tags = blobTags ?? null; + if (sourceChannel.StorageContext != null && sourceChannel.StorageContext.Track2OauthToken != null) + { + string oauthToken = sourceChannel.StorageContext.Track2OauthToken.GetToken(new TokenRequestContext(), this.CmdletCancellationToken).Token; + options.SourceAuthentication = new HttpAuthorization("Bearer", oauthToken); + } + destBlobClient.SyncCopyFromUri(srcUri, options, this.CmdletCancellationToken); // Set rehydrate priority @@ -542,12 +569,19 @@ private async Task CopyFromUri(long taskId, IStorageBlobManagement destChannel, progressHandler.Report(copyoffset); foreach (string id in blockIDs) { + Track2Models.StageBlockFromUriOptions stageBlockOptions = new Track2Models.StageBlockFromUriOptions(); long blocksize = blockLength; if (copyoffset + blocksize > srcProperties.ContentLength) { blocksize = srcProperties.ContentLength - copyoffset; } - destBlockBlob.StageBlockFromUri(srcUri, id, new global::Azure.HttpRange(copyoffset, blocksize), null, null, null, cancellationToken: this.CmdletCancellationToken); + stageBlockOptions.SourceRange = new global::Azure.HttpRange(copyoffset, blocksize); + if (sourceChannel.StorageContext != null && sourceChannel.StorageContext.Track2OauthToken != null) + { + string oauthToken = sourceChannel.StorageContext.Track2OauthToken.GetToken(null, this.CmdletCancellationToken).TokenValue; + stageBlockOptions.SourceAuthentication = new HttpAuthorization("Bearer", oauthToken); + } + destBlockBlob.StageBlockFromUri(srcUri, id, stageBlockOptions, cancellationToken: this.CmdletCancellationToken); copyoffset += blocksize; progressHandler.Report(copyoffset); diff --git a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlobContent.cs b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlobContent.cs index 816064a16167..740a76bd8127 100644 --- a/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlobContent.cs +++ b/src/Storage/Storage/Blob/Cmdlet/GetAzureStorageBlobContent.cs @@ -262,7 +262,7 @@ internal void GetBlobContent(CloudBlobContainer container, string blobName, stri ValidatePipelineCloudBlobContainer(container); - if (UseTrack2Sdk()) + if (UseTrack2Sdk() || IsSasTokenWithOAuth(container)) { BlobContainerClient track2container = AzureStorageContainer.GetTrack2BlobContainerClient(container, Channel.StorageContext, ClientOptions); BlobBaseClient blobClient = track2container.GetBlobBaseClient(blobName); @@ -278,6 +278,19 @@ internal void GetBlobContent(CloudBlobContainer container, string blobName, stri } } + private bool IsSasTokenWithOAuth(CloudBlobContainer container) + { + if (container.ServiceClient.Credentials.IsSAS) //SAS + { + if (Channel.StorageContext.Track2OauthToken != null) + { + return true; + } + } + + return false; + } + /// /// get blob content /// @@ -543,7 +556,7 @@ public override void ExecuteCmdlet() case BlobParameterSet: if (ShouldProcess(CloudBlob.Name, "Download")) { - if (!(CloudBlob is InvalidCloudBlob) && !UseTrack2Sdk()) + if (!(CloudBlob is InvalidCloudBlob) && !UseTrack2Sdk() && !IsSasTokenWithOAuth(CloudBlob.Container)) { GetBlobContent(CloudBlob, FileName, true); } diff --git a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageBlobSasToken.cs b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageBlobSasToken.cs index 4fcb24f2a020..9b1f3e381883 100644 --- a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageBlobSasToken.cs +++ b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageBlobSasToken.cs @@ -109,6 +109,12 @@ public string Policy [ValidateNotNullOrEmpty] public string Permission { get; set; } + [Parameter( + Mandatory = false, + HelpMessage = "Delegation object id")] + [ValidateNotNullOrEmpty] + public string DelegationObjectID { get; set; } + [Parameter(Mandatory = false, HelpMessage = "Protocol can be used in the request with this SAS token.")] [ValidateNotNull] public SharedAccessProtocol? Protocol { get; set; } @@ -176,7 +182,9 @@ public override void ExecuteCmdlet() // When the input context is Oauth bases, can't generate normal SAS, but UserDelegationSas bool generateUserDelegationSas = false; - if (Channel != null && Channel.StorageContext != null && Channel.StorageContext.StorageAccount.Credentials != null && Channel.StorageContext.StorageAccount.Credentials.IsToken) + if (Channel != null && Channel.StorageContext != null && ( + (Channel.StorageContext.StorageAccount.Credentials != null && Channel.StorageContext.StorageAccount.Credentials.IsToken) + || (Channel.StorageContext.Track2OauthToken != null))) { if (ShouldProcess(blob.Name, "Generate User Delegation SAS, since input Storage Context is OAuth based.")) { @@ -211,7 +219,7 @@ public override void ExecuteCmdlet() } //Create SAS builder - BlobSasBuilder sasBuilder = SasTokenHelper.SetBlobSasBuilder_FromBlob(blobClient, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol, this.EncryptionScope); + BlobSasBuilder sasBuilder = SasTokenHelper.SetBlobSasBuilder_FromBlob(blobClient, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol, this.EncryptionScope, this.DelegationObjectID); //Create SAS and output string sasToken = SasTokenHelper.GetBlobSharedAccessSignature(Channel.StorageContext, sasBuilder, generateUserDelegationSas, ClientOptions, CmdletCancellationToken); diff --git a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainerSasToken.cs b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainerSasToken.cs index 7dbcf4c4c7a9..bd516689ff3f 100644 --- a/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainerSasToken.cs +++ b/src/Storage/Storage/Blob/Cmdlet/NewAzureStorageContainerSasToken.cs @@ -67,6 +67,12 @@ public string Policy [ValidateNotNullOrEmpty] public string Permission { get; set; } + [Parameter( + Mandatory = false, + HelpMessage = "Delegation object id")] + [ValidateNotNullOrEmpty] + public string DelegationObjectID { get; set; } + [Parameter(Mandatory = false, HelpMessage = "Protocol can be used in the request with this SAS token.")] [ValidateNotNull] public SharedAccessProtocol? Protocol { get; set; } @@ -151,7 +157,7 @@ public override void ExecuteCmdlet() } //Create SAS builder - BlobSasBuilder sasBuilder = SasTokenHelper.SetBlobSasBuilder_FromContainer(container, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol, this.EncryptionScope); + BlobSasBuilder sasBuilder = SasTokenHelper.SetBlobSasBuilder_FromContainer(container, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol, this.EncryptionScope, this.DelegationObjectID); //Create SAS and output it string sasToken = SasTokenHelper.GetBlobSharedAccessSignature(Channel.StorageContext, sasBuilder, generateUserDelegationSas, ClientOptions, CmdletCancellationToken); diff --git a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs index 123ec9610726..c036cf7ef7ae 100644 --- a/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs +++ b/src/Storage/Storage/Blob/Cmdlet/SetAzureStorageBlobContent.cs @@ -395,7 +395,7 @@ protected override void DoEndProcessing() Tuple uploadRequest = UploadRequests.DequeueRequest(); IStorageBlobManagement localChannel = Channel; Func taskGenerator; - if (!UseTrack2Sdk()) + if (!(UseTrack2Sdk() || localChannel.IsSasWithOAuthCredential())) { //Upload with DMlib taskGenerator = (taskId) => Upload2Blob(taskId, localChannel, uploadRequest.Item1, uploadRequest.Item2); @@ -447,9 +447,10 @@ internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagemen { options = SetClientOptionsWithEncryptionScope(this.EncryptionScope); } + BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options); if (this.Force.IsPresent - || !blob.Exists() + || !blobClient.Exists() || ShouldContinue(string.Format(Resources.OverwriteConfirmation, blob.Uri), null)) { // Prepare blob Properties, MetaData, accessTier @@ -484,7 +485,6 @@ internal virtual async Task UploadBlobwithSdk(long taskId, IStorageBlobManagemen //block blob if (string.Equals(blobType, BlockBlobType, StringComparison.InvariantCultureIgnoreCase)) { - BlobClient blobClient = GetTrack2BlobClient(blob, localChannel.StorageContext, options); outputBlobClient = blobClient; StorageTransferOptions trasnferOption = new StorageTransferOptions() { MaximumConcurrency = this.GetCmdletConcurrency() }; BlobUploadOptions uploadOptions = new BlobUploadOptions(); diff --git a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs index 43f0e54ad9c5..cb955e4a33bf 100644 --- a/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs +++ b/src/Storage/Storage/Blob/StorageCloudBlobCmdletBase.cs @@ -921,7 +921,14 @@ public static BlobClient GetTrack2BlobClient(CloudBlob cloubBlob, AzureStorageCo { fullUri = fullUri + "?" + sas; } - blobClient = new BlobClient(new Uri(fullUri), options); + if (context.Track2OauthToken != null) + { + blobClient = new BlobClient(new Uri(fullUri), context.Track2OauthToken, options); + } + else + { + blobClient = new BlobClient(new Uri(fullUri), options); + } } else if (cloubBlob.ServiceClient.Credentials.IsSharedKey) //Shared Key { diff --git a/src/Storage/Storage/Common/AzureStorageBlob.cs b/src/Storage/Storage/Common/AzureStorageBlob.cs index ed0ebe64dac3..f47f31f43abe 100644 --- a/src/Storage/Storage/Common/AzureStorageBlob.cs +++ b/src/Storage/Storage/Common/AzureStorageBlob.cs @@ -451,7 +451,8 @@ public void FetchAttributes() public static BlobClient GetTrack2BlobClient(CloudBlob cloubBlob, AzureStorageContext context, BlobClientOptions options = null) { BlobClient blobClient; - if (cloubBlob.ServiceClient.Credentials.IsToken) //Oauth + if (cloubBlob.ServiceClient.Credentials.IsToken + || context.Track2OauthToken != null) //Oauth { if (context == null) { @@ -474,7 +475,15 @@ public static BlobClient GetTrack2BlobClient(CloudBlob cloubBlob, AzureStorageCo { fullUri = fullUri + "?" + sas; } - blobClient = new BlobClient(new Uri(fullUri), options); + if (context.Track2OauthToken != null) + { + blobClient = new BlobClient(new Uri(fullUri), context.Track2OauthToken, options); + } + else + { + + blobClient = new BlobClient(new Uri(fullUri), options); + } } else if (cloubBlob.ServiceClient.Credentials.IsSharedKey) //Shared Key { diff --git a/src/Storage/Storage/Common/AzureStorageContainer.cs b/src/Storage/Storage/Common/AzureStorageContainer.cs index eee568794b2a..30946e28527c 100644 --- a/src/Storage/Storage/Common/AzureStorageContainer.cs +++ b/src/Storage/Storage/Common/AzureStorageContainer.cs @@ -262,7 +262,14 @@ public static BlobContainerClient GetTrack2BlobContainerClient(CloudBlobContaine string fullUri = cloubContainer.Uri.ToString(); string sas = Util.GetSASStringWithoutQuestionMark(cloubContainer.ServiceClient.Credentials.SASToken); fullUri = fullUri + "?" + sas; - blobContainerClient = new BlobContainerClient(new Uri(fullUri), options); + if (context.Track2OauthToken != null) + { + blobContainerClient = new BlobContainerClient(new Uri(fullUri), context.Track2OauthToken, options); + } + else + { + blobContainerClient = new BlobContainerClient(new Uri(fullUri), options); + } } else if (cloubContainer.ServiceClient.Credentials.IsSharedKey) //Shared Key { diff --git a/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs b/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs index 859d9acdbfe7..ffe4bfbac382 100644 --- a/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs +++ b/src/Storage/Storage/Common/Cmdlet/NewAzureStorageContext.cs @@ -178,6 +178,16 @@ public SwitchParameter Anonymous private bool isAnonymous; + [Parameter(HelpMessage = "Use OAuth storage account", Mandatory = false, ParameterSetName = SasTokenParameterSet)] + public SwitchParameter SASTokenWithConnectedAccount + { + get { return sasTokenWithConnectedAcount; } + set { sasTokenWithConnectedAcount = value; } + } + + private bool sasTokenWithConnectedAcount = false; + + [Parameter(HelpMessage = "Use OAuth storage account", ParameterSetName = SasTokenParameterSet)] [Parameter(HelpMessage = "Use OAuth storage account", Mandatory = false, ParameterSetName = OAuthParameterSet)] [Parameter(HelpMessage = "Use OAuth storage account", Mandatory = false, ParameterSetName = OAuthEnvironmentParameterSet)] [Parameter(HelpMessage = "Use OAuth storage account", Mandatory = false, ParameterSetName = OAuthServiceEndpointParameterSet)] @@ -639,7 +649,7 @@ public override void ExecuteCmdlet() throw new ArgumentException(Resources.DefaultStorageCredentialsNotFound); } - AzureStorageContext context = new AzureStorageContext(account, GetRealAccountName(StorageAccountName), DefaultContext, WriteDebug); + AzureStorageContext context = new AzureStorageContext(account, GetRealAccountName(StorageAccountName), this.sasTokenWithConnectedAcount, DefaultContext, WriteDebug); if (this.EnableFileBackupRequestIntent.IsPresent) { context.ShareTokenIntent = ShareTokenIntent.Backup; diff --git a/src/Storage/Storage/Common/SasTokenHelper.cs b/src/Storage/Storage/Common/SasTokenHelper.cs index bea108f6fff0..f543050ad364 100644 --- a/src/Storage/Storage/Common/SasTokenHelper.cs +++ b/src/Storage/Storage/Common/SasTokenHelper.cs @@ -14,23 +14,24 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common { - using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; - using Microsoft.Azure.Storage; - using Microsoft.Azure.Storage.Blob; - using XTable = Microsoft.Azure.Cosmos.Table; using System; using System.Collections.Generic; - using global::Azure.Storage.Sas; - using global::Azure.Storage.Blobs.Specialized; - using global::Azure.Storage.Blobs.Models; using System.Threading; - using global::Azure.Storage.Blobs; + using global::Azure.Core; using global::Azure.Storage; + using global::Azure.Storage.Blobs; + using global::Azure.Storage.Blobs.Models; + using global::Azure.Storage.Blobs.Specialized; using global::Azure.Storage.Files.DataLake; using global::Azure.Storage.Files.Shares; using global::Azure.Storage.Files.Shares.Models; - using global::Azure.Storage.Queues.Models; using global::Azure.Storage.Queues; + using global::Azure.Storage.Queues.Models; + using global::Azure.Storage.Sas; + using Microsoft.Azure.Storage; + using Microsoft.Azure.Storage.Blob; + using Microsoft.WindowsAzure.Commands.Storage.Model.Contract; + using XTable = Microsoft.Azure.Cosmos.Table; internal class SasTokenHelper { @@ -275,7 +276,8 @@ public static QueueSasBuilder SetQueueSasbuilder(QueueClient queue, DateTime? startTime = null, DateTime? expiryTime = null, string iPAddressOrRange = null, - string protocol = null) + string protocol = null, + string delegatedUserObjectId = null) { QueueSasBuilder sasBuilder = new QueueSasBuilder { @@ -371,6 +373,10 @@ public static QueueSasBuilder SetQueueSasbuilder(QueueClient queue, sasBuilder.Protocol = SasProtocol.Https; } } + if (delegatedUserObjectId != null) + { + sasBuilder.DelegatedUserObjectId = delegatedUserObjectId; + } return sasBuilder; } @@ -508,12 +514,31 @@ public static string GetFileSharedAccessSignature(AzureStorageContext context, S /// /// Get Queue SAS string /// - public static string GetQueueSharedAccessSignature(AzureStorageContext context, QueueSasBuilder sasBuilder, CancellationToken cancellationToken) + public static string GetQueueSharedAccessSignature(AzureStorageContext context, QueueSasBuilder sasBuilder, bool generateUserDelegationSas, CancellationToken cancellationToken) { if (context != null && context.StorageAccount != null && context.StorageAccount.Credentials != null && context.StorageAccount.Credentials.IsSharedKey) { return sasBuilder.ToSasQueryParameters(new StorageSharedKeyCredential(context.StorageAccountName, context.StorageAccount.Credentials.ExportBase64EncodedKey())).ToString(); } + + if (generateUserDelegationSas) + { + if (context.StorageAccountName.StartsWith("[")) + { + throw new InvalidOperationException("Please provide '-Context' as a storage context created by cmdlet `New-AzStorageContext` with parameters include '-StorageAccountName'."); + } + global::Azure.Storage.Queues.Models.UserDelegationKey userDelegationKey = null; + QueueServiceClient oauthService = new QueueServiceClient(context.StorageAccount.QueueEndpoint, context.Track2OauthToken); + + Util.ValidateUserDelegationKeyStartEndTime(sasBuilder.StartsOn, sasBuilder.ExpiresOn); + + userDelegationKey = oauthService.GetUserDelegationKey( + startsOn: sasBuilder.StartsOn == DateTimeOffset.MinValue || sasBuilder.StartsOn == null ? DateTimeOffset.UtcNow : sasBuilder.StartsOn.ToUniversalTime(), + expiresOn: sasBuilder.ExpiresOn.ToUniversalTime(), + cancellationToken: cancellationToken); + + return sasBuilder.ToSasQueryParameters(userDelegationKey, context.StorageAccountName).ToString(); + } else { throw new InvalidOperationException("Create Queue service SAS only supported with SharedKey credential."); @@ -531,7 +556,8 @@ public static BlobSasBuilder SetBlobSasBuilder_FromBlob(BlobBaseClient blobClien DateTime? ExpiryTime = null, string iPAddressOrRange = null, SharedAccessProtocol? Protocol = null, - string EncryptionScope = null) + string EncryptionScope = null, + string DelegatedUserObjectId = null) { BlobSasBuilder sasBuilder = SetBlobSasBuilder(blobClient.BlobContainerName, blobClient.Name, @@ -541,7 +567,8 @@ public static BlobSasBuilder SetBlobSasBuilder_FromBlob(BlobBaseClient blobClien ExpiryTime, iPAddressOrRange, Protocol, - EncryptionScope); + EncryptionScope, + DelegatedUserObjectId); if (Util.GetVersionIdFromBlobUri(blobClient.Uri) != null) { sasBuilder.BlobVersionId = Util.GetVersionIdFromBlobUri(blobClient.Uri); @@ -563,7 +590,8 @@ public static BlobSasBuilder SetBlobSasBuilder_FromContainer(BlobContainerClient DateTime? ExpiryTime = null, string iPAddressOrRange = null, SharedAccessProtocol? Protocol = null, - string EncryptionScope = null) + string EncryptionScope = null, + string DelegatedUserObjectId = null) { BlobSasBuilder sasBuilder = SetBlobSasBuilder(container.Name, null, @@ -573,7 +601,8 @@ public static BlobSasBuilder SetBlobSasBuilder_FromContainer(BlobContainerClient ExpiryTime, iPAddressOrRange, Protocol, - EncryptionScope); + EncryptionScope, + DelegatedUserObjectId); return sasBuilder; } @@ -588,7 +617,8 @@ public static BlobSasBuilder SetBlobSasBuilder(string containerName, DateTime? ExpiryTime = null, string iPAddressOrRange = null, SharedAccessProtocol? Protocol = null, - string EncryptionScope = null) + string EncryptionScope = null, + string DelegatedUserObjectId = null) { BlobSasBuilder sasBuilder; if (signedIdentifier != null) // Use save access policy @@ -694,6 +724,10 @@ public static BlobSasBuilder SetBlobSasBuilder(string containerName, { sasBuilder.EncryptionScope = EncryptionScope; } + if(DelegatedUserObjectId != null) + { + sasBuilder.DelegatedUserObjectId = DelegatedUserObjectId; + } return sasBuilder; } diff --git a/src/Storage/Storage/Common/Util.cs b/src/Storage/Storage/Common/Util.cs index 9b030cf59b47..3278c17c6df2 100644 --- a/src/Storage/Storage/Common/Util.cs +++ b/src/Storage/Storage/Common/Util.cs @@ -487,20 +487,48 @@ public static BlobBaseClient GetTrack2BlobClient(Uri blobUri, AzureStorageContex { if (blobType == null) { - blobClient = new BlobBaseClient(blobUri, options); + if (context.Track2OauthToken != null) + { + blobClient = new BlobBaseClient(blobUri, context.Track2OauthToken, options); + } + else + { + blobClient = new BlobBaseClient(blobUri, options); + } } else { switch (blobType.Value) { case global::Azure.Storage.Blobs.Models.BlobType.Page: - blobClient = new PageBlobClient(blobUri, options); + if (context.Track2OauthToken != null) + { + blobClient = new PageBlobClient(blobUri, context.Track2OauthToken, options); + } + else + { + blobClient = new PageBlobClient(blobUri, options); + } break; case global::Azure.Storage.Blobs.Models.BlobType.Append: - blobClient = new AppendBlobClient(blobUri, options); + if (context.Track2OauthToken != null) + { + blobClient = new AppendBlobClient(blobUri, context.Track2OauthToken, options); + } + else + { + blobClient = new AppendBlobClient(blobUri, options); + } break; default: //Block - blobClient = new BlockBlobClient(blobUri, options); + if (context.Track2OauthToken != null) + { + blobClient = new BlockBlobClient(blobUri, context.Track2OauthToken, options); + } + else + { + blobClient = new BlockBlobClient(blobUri, options); + } break; } } @@ -525,11 +553,19 @@ public static BlobServiceClient GetTrack2BlobServiceClient(AzureStorageContext c string connectionString = context.ConnectionString; // remove the "?" at the begin of SAS if any + bool withSasToken = false; if (context != null && context.StorageAccount != null && context.StorageAccount.Credentials != null && context.StorageAccount.Credentials.IsSAS) { + withSasToken = true; connectionString = connectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); } + blobServiceClient = new BlobServiceClient(connectionString, options); + + if (withSasToken && context.Track2OauthToken != null) + { + blobServiceClient = new BlobServiceClient(blobServiceClient.Uri, context.Track2OauthToken, options); + } } return blobServiceClient; } @@ -785,8 +821,17 @@ public static QueueServiceClient GetTrack2QueueServiceClient(AzureStorageContext if (context != null && context.StorageAccount != null && context.StorageAccount.Credentials != null && context.StorageAccount.Credentials.IsSAS) { connectionString = connectionString.Replace("SharedAccessSignature=?", "SharedAccessSignature="); + queueServiceClient = new QueueServiceClient(connectionString, options); + + if (context != null && context.Track2OauthToken != null) + { + queueServiceClient = new QueueServiceClient(queueServiceClient.Uri, context.Track2OauthToken, options); + } + } + else + { + queueServiceClient = new QueueServiceClient(connectionString, options); } - queueServiceClient = new QueueServiceClient(connectionString, options); } return queueServiceClient; } diff --git a/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs index d5a1f4ca6cc0..bbf5e6e72bbd 100644 --- a/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs +++ b/src/Storage/Storage/DatalakeGen2/Cmdlet/GetAzDataLakeGen2ChildItem.cs @@ -123,7 +123,13 @@ public override void ExecuteCmdlet() Page page; do { - IEnumerator> enumerator = fileSystem.GetPaths(this.Path, this.Recurse, this.OutputUserPrincipalName.IsPresent) + DataLakeGetPathsOptions options = new DataLakeGetPathsOptions() + { + Path = this.Path, + Recursive = this.Recurse, + UserPrincipalName = this.OutputUserPrincipalName.IsPresent + }; + IEnumerator> enumerator = fileSystem.GetPaths(options) .AsPages(this.ContinuationToken, listCount) .GetEnumerator(); diff --git a/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs b/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs index 9a8ba47a6650..828e9f45482c 100644 --- a/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs +++ b/src/Storage/Storage/Model/Contract/IStorageBlobManagement.cs @@ -30,6 +30,8 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Model.Contract /// public interface IStorageBlobManagement : IStorageManagement { + bool IsSasWithOAuthCredential(); + /// /// Get a list of cloudblobcontainer in azure /// diff --git a/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs b/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs index 9261b69c2e3b..524b212325d9 100644 --- a/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs +++ b/src/Storage/Storage/Model/Contract/StorageBlobManagement.cs @@ -66,6 +66,10 @@ private CloudBlobClient BlobClient return this.blobClient; } } + public bool IsSasWithOAuthCredential() + { + return this.BlobClient.Credentials.IsSAS && this.StorageContext.Track2OauthToken != null; + } /// /// The azure storage context associated with this IStorageBlobManagement diff --git a/src/Storage/Storage/Queue/Cmdlet/NewAzureStorageQueueSasToken.cs b/src/Storage/Storage/Queue/Cmdlet/NewAzureStorageQueueSasToken.cs index 3817d0cf5310..ef6a7acb344f 100644 --- a/src/Storage/Storage/Queue/Cmdlet/NewAzureStorageQueueSasToken.cs +++ b/src/Storage/Storage/Queue/Cmdlet/NewAzureStorageQueueSasToken.cs @@ -61,6 +61,12 @@ public string Policy [ValidateNotNullOrEmpty] public string Permission { get; set; } + [Parameter( + Mandatory = false, + HelpMessage = "Delegation object id")] + [ValidateNotNullOrEmpty] + public string DelegationObjectID { get; set; } + [Parameter(Mandatory = false, HelpMessage = "Protocol can be used in the request with this SAS token.")] [ValidateSet("HttpsOnly", "HttpsOrHttp", IgnoreCase = true),] public string Protocol { get; set; } @@ -110,6 +116,24 @@ public override void ExecuteCmdlet() { if (String.IsNullOrEmpty(Name)) return; + // When the input context is Oauth bases, can't generate normal SAS, but UserDelegationSas + bool generateUserDelegationSas = false; + if (Channel != null && Channel.StorageContext != null && Channel.StorageContext.StorageAccount.Credentials != null && Channel.StorageContext.StorageAccount.Credentials.IsToken) + { + if (ShouldProcess(Name, "Generate User Delegation SAS, since input Storage Context is OAuth based.")) + { + generateUserDelegationSas = true; + if (!string.IsNullOrEmpty(accessPolicyIdentifier) || !string.IsNullOrEmpty(this.Policy)) + { + throw new ArgumentException("When input Storage Context is OAuth based, Saved Policy is not supported.", "Policy"); + } + } + else + { + return; + } + } + QueueClient queueClient = Util.GetTrack2QueueClient(this.Name, (AzureStorageContext)this.Context, this.ClientOptions); QueueSignedIdentifier identifier = null; if (!string.IsNullOrEmpty(this.Policy)) @@ -117,8 +141,8 @@ public override void ExecuteCmdlet() identifier = SasTokenHelper.GetQueueSignedIdentifier(queueClient, this.Policy, CmdletCancellationToken); } - QueueSasBuilder sasBuilder = SasTokenHelper.SetQueueSasbuilder(queueClient, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol); - string sasToken = SasTokenHelper.GetQueueSharedAccessSignature((AzureStorageContext)this.Context, sasBuilder, CmdletCancellationToken); + QueueSasBuilder sasBuilder = SasTokenHelper.SetQueueSasbuilder(queueClient, identifier, this.Permission, this.StartTime, this.ExpiryTime, this.IPAddressOrRange, this.Protocol, this.DelegationObjectID); + string sasToken = SasTokenHelper.GetQueueSharedAccessSignature((AzureStorageContext)this.Context, sasBuilder, generateUserDelegationSas, CmdletCancellationToken); // remove prefix "?" of SAS if any sasToken = Util.GetSASStringWithoutQuestionMark(sasToken); diff --git a/src/Storage/Storage/Storage.csproj b/src/Storage/Storage/Storage.csproj index 25433d9563af..0a4d50d9e49d 100644 --- a/src/Storage/Storage/Storage.csproj +++ b/src/Storage/Storage/Storage.csproj @@ -13,10 +13,10 @@ - - - - + + + + diff --git a/src/lib/cgmanifest.json b/src/lib/cgmanifest.json index fb172140cf5d..e4f64c8b61e3 100644 --- a/src/lib/cgmanifest.json +++ b/src/lib/cgmanifest.json @@ -88,7 +88,7 @@ "type": "nuget", "nuget": { "name": "Azure.Core", - "version": "1.47.3" + "version": "1.50.0" } } }, @@ -178,7 +178,7 @@ "type": "nuget", "nuget": { "name": "System.ClientModel", - "version": "1.6.1" + "version": "1.8.0" } } }, diff --git a/src/lib/manifest.json b/src/lib/manifest.json index 0b31a449b4e2..a902489bee65 100644 --- a/src/lib/manifest.json +++ b/src/lib/manifest.json @@ -1,7 +1,7 @@ [ { "PackageName": "Azure.Core", - "PackageVersion": "1.47.3", + "PackageVersion": "1.50.0", "TargetFramework": "netstandard2.0", "WindowsPowerShell": true, "PowerShell7Plus": true @@ -72,7 +72,7 @@ }, { "PackageName": "System.ClientModel", - "PackageVersion": "1.6.1", + "PackageVersion": "1.8.0", "TargetFramework": "netstandard2.0", "WindowsPowerShell": true, "PowerShell7Plus": true diff --git a/src/lib/netstandard2.0/Azure.Core.dll b/src/lib/netstandard2.0/Azure.Core.dll index 9e01bb3ec3be..110eeca767a5 100644 Binary files a/src/lib/netstandard2.0/Azure.Core.dll and b/src/lib/netstandard2.0/Azure.Core.dll differ diff --git a/src/lib/netstandard2.0/System.ClientModel.dll b/src/lib/netstandard2.0/System.ClientModel.dll index 233dff28ff25..0679f97eb55f 100644 Binary files a/src/lib/netstandard2.0/System.ClientModel.dll and b/src/lib/netstandard2.0/System.ClientModel.dll differ diff --git a/tools/Common.Netcore.Dependencies.targets b/tools/Common.Netcore.Dependencies.targets index bf3bfff8e8e4..2ea25fb4890f 100644 --- a/tools/Common.Netcore.Dependencies.targets +++ b/tools/Common.Netcore.Dependencies.targets @@ -22,7 +22,7 @@ - +