diff --git a/SamplesV1/ADFCustomActivityRunner/ADFCustomActivityRunner.sln b/SamplesV1/ADFCustomActivityRunner/ADFCustomActivityRunner.sln deleted file mode 100644 index f26aa650..00000000 --- a/SamplesV1/ADFCustomActivityRunner/ADFCustomActivityRunner.sln +++ /dev/null @@ -1,34 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25420.1 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CustomActivityRunner", "CustomActivityRunner\CustomActivityRunner.csproj", "{897254B5-674B-4D87-8D9D-917EF43DB182}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotNetActivityRunner", "DotNetActivityRunner\DotNetActivityRunner.csproj", "{7808BBFC-83DC-49D4-BDAE-467D6044FDA1}" -EndProject -Project("{FF286327-C783-4F7A-AB73-9BCBAD0D4460}") = "ADFCustomActivityRunner", "NuGetPackage\ADFCustomActivityRunner.nuproj", "{BB5B7DE2-2A1A-4837-80C1-11C4693C28EC}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {897254B5-674B-4D87-8D9D-917EF43DB182}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {897254B5-674B-4D87-8D9D-917EF43DB182}.Debug|Any CPU.Build.0 = Debug|Any CPU - {897254B5-674B-4D87-8D9D-917EF43DB182}.Release|Any CPU.ActiveCfg = Release|Any CPU - {897254B5-674B-4D87-8D9D-917EF43DB182}.Release|Any CPU.Build.0 = Release|Any CPU - {7808BBFC-83DC-49D4-BDAE-467D6044FDA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7808BBFC-83DC-49D4-BDAE-467D6044FDA1}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7808BBFC-83DC-49D4-BDAE-467D6044FDA1}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7808BBFC-83DC-49D4-BDAE-467D6044FDA1}.Release|Any CPU.Build.0 = Release|Any CPU - {BB5B7DE2-2A1A-4837-80C1-11C4693C28EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BB5B7DE2-2A1A-4837-80C1-11C4693C28EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BB5B7DE2-2A1A-4837-80C1-11C4693C28EC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BB5B7DE2-2A1A-4837-80C1-11C4693C28EC}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/BlobUtilities.cs b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/BlobUtilities.cs deleted file mode 100644 index f8d650eb..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/BlobUtilities.cs +++ /dev/null @@ -1,191 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Azure.Management.DataFactories.Runtime; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; - -namespace CustomActivityRunner -{ - public class BlobUtilities - { - private IActivityLogger logger; - private string connectionString; - private string folderPath; - - public BlobUtilities(IActivityLogger logger, string connectionString, string folderPath) - { - this.logger = logger; - this.connectionString = connectionString; - this.folderPath = folderPath; - } - - public string DownLoadFile(string blobFileName) - { - logger.Write($"Downloading file '{blobFileName}' from blob '{folderPath}'"); - - var blockBlob = GetCloudBlockBlob(blobFileName); - string localFile = Path.Combine(Path.GetTempPath(), blobFileName); - - if (!blockBlob.Exists()) - { - return null; - } - - blockBlob.DownloadToFile(localFile, FileMode.Create); - - return localFile; - } - - public string DownLoadLatestFile() - { - CloudBlockBlob latestBlob = GetLatestBlob(); - - if (latestBlob == null) - return null; - - string filename; - - // If the blob is embedded in a folder heirarchy find the name of the actual file - if(latestBlob.Name.Contains("/")) - filename = latestBlob.Name.Substring(latestBlob.Name.LastIndexOf('/')+1); - else - filename = latestBlob.Name; - - - string localFile = Path.Combine(GetTemporaryDirectory(), filename); - - logger.Write($"Downloading latest blob file '{latestBlob.Name}' from blob folder '{folderPath}'"); - - latestBlob.DownloadToFile(localFile, FileMode.Create); - - return localFile; - } - - public void UploadFile(string localFilePath) - { - logger.Write($"Uploading file '{localFilePath}' to blob '{folderPath}'"); - - var blockBlob = GetCloudBlockBlob(Path.GetFileName(localFilePath), true); - blockBlob.UploadFromFile(localFilePath, FileMode.Open); - } - - public async Task DownLoadFileAsync(string fileName) - { - var blockBlob = GetCloudBlockBlob(fileName); - string localFile = Path.Combine(Path.GetTempPath(), fileName); - await blockBlob.DownloadToFileAsync(localFile, FileMode.Create); - - return localFile; - } - - public string GetLatestBlobFileName() - { - CloudBlockBlob latestBlob = GetLatestBlob(); - - return Path.GetFileNameWithoutExtension(latestBlob?.Name); - } - - public void RemoveFilesFromBlob(string fileName = null) - { - var container = GetCloudBlobContainer(); - - var blobs = container.ListBlobs(); - - // Optionally delete just a single file - if (fileName != null) - { - blobs = container.ListBlobs().Where(x => x.Uri.ToString().Contains(fileName)); - } - - foreach (CloudBlockBlob blob in blobs) - { - logger.Write($"Removing blob '{blob.Name}' from storage '{folderPath}'"); - blob.DeleteIfExists(); - } - } - - - private CloudBlockBlob GetCloudBlockBlob(string fileName, bool createContainer = false) - { - var container = GetCloudBlobContainer(); - - if (createContainer) - { - //Create a new container, if it does not exist - container.CreateIfNotExists(); - } - - // Get the block blob - CloudBlockBlob blockBlob = container.GetBlockBlobReference(fileName); - - return blockBlob; - } - - private CloudBlobContainer GetCloudBlobContainer(string inputPath = null) - { - var path = inputPath ?? folderPath; - - CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(connectionString); - CloudBlobClient blobClient = inputStorageAccount.CreateCloudBlobClient(); - CloudBlobContainer container = blobClient.GetContainerReference(path); - return container; - } - - public CloudBlockBlob GetLatestBlob() - { - var container = GetCloudBlobContainer(); - var blobs = container.ListBlobs(useFlatBlobListing:true); - - - var listBlobItems = blobs as IList ?? blobs.ToList(); - CloudBlockBlob latestBlob = !listBlobItems.Any() ? null : - listBlobItems.OrderByDescending(x => ((CloudBlockBlob)x).Properties.LastModified).First() as CloudBlockBlob; - - return latestBlob; - } - - private string GetTemporaryDirectory() - { - string tempDirectory = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); - Directory.CreateDirectory(tempDirectory); - return tempDirectory; - } - - public void ArchiveFile(string archiveContainer, string path) - { - var blobName = DateTime.Now.ToString("s").ToLower() + ": " + Path.GetFileName(path).ToLower(); - - CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(connectionString); - CloudBlobClient blobClient = inputStorageAccount.CreateCloudBlobClient(); - CloudBlobContainer container = blobClient.GetContainerReference(archiveContainer); - - - var cloudBlock = container.GetBlockBlobReference(blobName); - - using (var fileStream = File.OpenRead(path)) - { - cloudBlock.UploadFromStream(fileStream); - } - } - - public void CopyToNewBlobLocation(string storageConnectionString, string filePath, string folderName) - { - var blobName = Path.GetFileName(filePath).ToLower(); - - CloudStorageAccount inputStorageAccount = CloudStorageAccount.Parse(storageConnectionString); - CloudBlobClient blobClient = inputStorageAccount.CreateCloudBlobClient(); - CloudBlobContainer container = blobClient.GetContainerReference(folderName); - - var cloudBlock = container.GetBlockBlobReference(blobName); - - using (var fileStream = File.OpenRead(filePath)) - { - cloudBlock.UploadFromStream(fileStream); - } - } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityAttribute.cs b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityAttribute.cs deleted file mode 100644 index 1200ae64..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityAttribute.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using NUnit.Framework; - -namespace CustomActivityRunner -{ - /// - /// This attribute when applied to a custom DotNetActivity allows it to be debugged by clicking on the debug button that appears beside the RunActivity method. - /// - public class CustomActivityAttribute : TestAttribute - { - /// - /// Specify the location of the pipeline json file relative to the project hosting the custom DotNetActivity. - /// - public string PipelineLocation { get; set; } - - /// - /// The name of the activity to debug. - /// - public string ActivityName { get; set; } - - /// - /// The name of the deployment configuration file you wish to use with the activity. This is optional. - /// - public string DeployConfig { get; set; } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityBase.cs b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityBase.cs deleted file mode 100644 index be0c2e61..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityBase.cs +++ /dev/null @@ -1,180 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ADF.DotNetActivityRunner; -using Microsoft.Azure.Management.DataFactories.Models; -using Microsoft.Azure.Management.DataFactories.Runtime; - -namespace CustomActivityRunner -{ - public abstract class CustomActivityBase : IDotNetActivity - { - public IEnumerable LinkedServices { get; private set; } - - public IEnumerable Datasets { get; private set; } - - public Activity Activity { get; private set; } - - public IActivityLogger Logger { get; private set; } - - private DotNetActivity typeProperties; - - public CustomActivityBase() - { - if (Debugger.IsAttached) - { - var attributes = this.GetType().GetMethod("RunActivity").CustomAttributes; - var customActivityAttribute = attributes.FirstOrDefault(x => x.AttributeType.Name == "CustomActivityAttribute"); - - string activityName = customActivityAttribute?.NamedArguments?.FirstOrDefault(x => x.MemberName == "ActivityName").TypedValue.Value?.ToString(); - string pipelineLocation = customActivityAttribute?.NamedArguments?.FirstOrDefault(x => x.MemberName == "PipelineLocation").TypedValue.Value?.ToString(); - string deployConfig = customActivityAttribute?.NamedArguments?.FirstOrDefault(x => x.MemberName == "DeployConfig").TypedValue.Value?.ToString(); - - - if (!string.IsNullOrEmpty(activityName) || !string.IsNullOrEmpty(pipelineLocation)) - { - string dataFactoryProjLocation = - Path.GetFullPath(Path.Combine(Directory.GetCurrentDirectory(), "..\\..", Path.GetDirectoryName(pipelineLocation))); - - DotNetActivityContext context = Runner.DeserializeActivity(Path.GetFileName(pipelineLocation), activityName, deployConfig, dataFactoryProjLocation); - - LinkedServices = context.LinkedServices; - Datasets = context.Datasets; - Activity = context.Activity; - Logger = context.Logger; - - typeProperties = Activity.TypeProperties as DotNetActivity; - } - else - { - throw new Exception($"The CustomActivity attribute needs to have the following properties populated: {nameof(CustomActivityAttribute.PipelineLocation)} and {nameof(CustomActivityAttribute.ActivityName)}"); - } - } - } - - public IDictionary Execute(IEnumerable linkedServices, IEnumerable datasets, Activity activity, IActivityLogger logger) - { - LinkedServices = linkedServices; - Datasets = datasets; - Activity = activity; - Logger = logger; - - typeProperties = Activity.TypeProperties as DotNetActivity; - - return RunActivity(); - } - - public abstract IDictionary RunActivity(); - - public string GetExtendedProperty(string name) - { - if (!typeProperties.ExtendedProperties.ContainsKey(name)) - { - throw new KeyNotFoundException($"The extended property '{name}' was not found in the extendedProperties section of the activity.'"); - } - - return typeProperties.ExtendedProperties[name]; - } - - public string GetInputSqlConnectionString() - { - string activityInputName = Activity.Inputs.First().Name; - return GetSqlConnectionString(activityInputName); - } - public string GetOutputSqlConnectionString() - { - string activityOutputName = Activity.Outputs.First().Name; - return GetSqlConnectionString(activityOutputName); - } - - public string GetSqlConnectionString(string datasetName) - { - Dataset dataset = Datasets.Single(x => x.Name == datasetName); - LinkedService linkedService = LinkedServices.First(x => x.Name == dataset.Properties.LinkedServiceName); - - if (linkedService.Properties.Type != "AzureSqlDatabase") - { - throw new Exception($"The linked service is of type '{linkedService.Properties.Type}'. It should be of type 'AzureSqlDatabase'."); - } - - AzureSqlDatabaseLinkedService sqlLinkedService = linkedService.Properties.TypeProperties as AzureSqlDatabaseLinkedService; - - if (sqlLinkedService == null) - { - throw new Exception($"Unable to find data set name '{datasetName}'."); - } - - string connectionString = sqlLinkedService.ConnectionString; - - if (string.IsNullOrEmpty(connectionString)) - { - throw new Exception($"Connection string for '{linkedService.Name}' linked service is empty."); - } - - return connectionString; - } - - public T GetLinkedService(string name) where T : class - { - int linkedServiceCount = LinkedServices.Count(x => x.Name == name); - if (linkedServiceCount == 0) - { - throw new Exception($"The linked service '{name}' was not found."); - } - if (linkedServiceCount > 1) - { - throw new Exception($"More than one linked service with name '{name}' were found. Only one should exist."); - } - - return LinkedServices.First(x => x.Name == name).Properties.TypeProperties as T; - } - - public T GetDataset(string name) where T : class - { - int datasetCount = Datasets.Count(x => x.Name == name); - if (datasetCount == 0) - { - throw new Exception($"The dataset '{name}' was not found."); - } - if (datasetCount > 1) - { - throw new Exception($"More than one datasets with name '{name}' were found. Only one should exist."); - } - - return Datasets.First(x => x.Name == name).Properties.TypeProperties as T; - } - - public string GetBlobFolderPath(string datasetName) - { - Dataset dataset = Datasets.Single(x => x.Name == datasetName); - AzureBlobDataset outputProps = (AzureBlobDataset)dataset.Properties.TypeProperties; - return outputProps.FolderPath; - } - - public BlobUtilities GetBlob(string datasetName) - { - Dataset dataset = Datasets.Single(x => x.Name == datasetName); - AzureBlobDataset outputProps = (AzureBlobDataset)dataset.Properties.TypeProperties; - AzureStorageLinkedService outputLinkedService = LinkedServices.First(x => x.Name == dataset.Properties.LinkedServiceName).Properties.TypeProperties as AzureStorageLinkedService; - string outputConnectionString = outputLinkedService?.ConnectionString; - BlobUtilities outputBlob = new BlobUtilities(Logger, outputConnectionString, outputProps.FolderPath); - return outputBlob; - } - - public Dictionary GetAllExtendedProperties() - { - DotNetActivity dotNetActivity = (DotNetActivity)Activity.TypeProperties; - - if (!dotNetActivity.ExtendedProperties.Any()) - { - throw new Exception($"No properties found in the extended properties section of the custom activity '{Activity.Name}'"); - } - - return dotNetActivity.ExtendedProperties as Dictionary; - } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityRunner.csproj b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityRunner.csproj deleted file mode 100644 index 4742793a..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/CustomActivityRunner.csproj +++ /dev/null @@ -1,166 +0,0 @@ - - - - - Debug - AnyCPU - {897254B5-674B-4D87-8D9D-917EF43DB182} - Library - Properties - CustomActivityRunner - CustomActivityRunner - v4.5.2 - 512 - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll - True - - - ..\packages\ADFSecurePublish.1.0.1.1\lib\net452\Microsoft.ADF.Deployment.AdfKeyVaultDeployment.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.NetFramework.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.1.0.0\lib\net45\Microsoft.Azure.KeyVault.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - True - - - ..\packages\Microsoft.Azure.Management.DataFactories.4.11.0\lib\net45\Microsoft.Azure.Management.DataFactories.dll - True - - - ..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll - True - - - ..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll - True - - - ..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll - True - - - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - True - - - ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll - True - - - ..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll - True - - - ..\packages\Newtonsoft.Json.Schema.2.0.7\lib\net45\Newtonsoft.Json.Schema.dll - True - - - ..\packages\NUnit.2.6.4\lib\nunit.framework.dll - True - - - - - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - True - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll - True - - - - ..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll - True - - - - - - - - - - - - - - - - - - - - - {7808bbfc-83dc-49d4-bdae-467d6044fda1} - DotNetActivityRunner - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/Properties/AssemblyInfo.cs b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/Properties/AssemblyInfo.cs deleted file mode 100644 index 1e4bd6e7..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("CustomActivityRunner")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("CustomActivityRunner")] -[assembly: AssemblyCopyright("Copyright © 2017")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("897254b5-674b-4d87-8d9d-917ef43db182")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/app.config b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/app.config deleted file mode 100644 index 110dafc1..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/app.config +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/packages.config b/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/packages.config deleted file mode 100644 index ed58a0a4..00000000 --- a/SamplesV1/ADFCustomActivityRunner/CustomActivityRunner/packages.config +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/ActivityLogger.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/ActivityLogger.cs deleted file mode 100644 index 8d0d09cc..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/ActivityLogger.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Microsoft.Azure.Management.DataFactories.Runtime; -using System; -using System.Diagnostics; - -namespace Microsoft.ADF.DotNetActivityRunner -{ - internal class ActivityLogger : IActivityLogger - { - internal ActivityLogger() - { - Trace.AutoFlush = true; - } - - public void Write(string format, params object[] args) - { - Console.WriteLine(format, args); - Trace.WriteLine(string.Format(format, args)); - } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityContext.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityContext.cs deleted file mode 100644 index dcca9289..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityContext.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Azure.Management.DataFactories.Models; -using Microsoft.Azure.Management.DataFactories.Runtime; - -namespace Microsoft.ADF.DotNetActivityRunner -{ - public class DotNetActivityContext - { - public List LinkedServices { get; set; } - public List Datasets { get; set; } - public Activity Activity { get; set; } - public IActivityLogger Logger { get; set; } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityRunner.csproj b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityRunner.csproj deleted file mode 100644 index 827327b5..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/DotNetActivityRunner.csproj +++ /dev/null @@ -1,171 +0,0 @@ - - - - - Debug - AnyCPU - {7808BBFC-83DC-49D4-BDAE-467D6044FDA1} - Library - Properties - Microsoft.ADF.DotNetActivityRunner - Microsoft.ADF.DotNetActivityRunner - v4.5.2 - 512 - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll - True - - - ..\packages\ADFSecurePublish.1.0.1.1\lib\net452\Microsoft.ADF.Deployment.AdfKeyVaultDeployment.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.NetFramework.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.1.0.0\lib\net45\Microsoft.Azure.KeyVault.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - True - - - ..\packages\Microsoft.Azure.Management.DataFactories.4.11.0\lib\net45\Microsoft.Azure.Management.DataFactories.dll - True - - - ..\packages\Microsoft.Data.Edm.5.7.0\lib\net40\Microsoft.Data.Edm.dll - True - - - ..\packages\Microsoft.Data.OData.5.7.0\lib\net40\Microsoft.Data.OData.dll - True - - - ..\packages\Microsoft.Data.Services.Client.5.7.0\lib\net40\Microsoft.Data.Services.Client.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll - True - - - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - True - - - ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll - True - - - ..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll - True - - - ..\packages\Newtonsoft.Json.Schema.2.0.7\lib\net45\Newtonsoft.Json.Schema.dll - True - - - - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - True - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll - True - - - - ..\packages\System.Spatial.5.7.0\lib\net40\System.Spatial.dll - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Activity.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Activity.cs deleted file mode 100644 index 08f0f44d..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Activity.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Newtonsoft.Json; -using System.Collections.Generic; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class Activity - { - [JsonProperty("name")] - internal string Name { get; set; } - - [JsonProperty("inputs")] - internal List Inputs { get; set; } - - [JsonProperty("outputs")] - internal List Outputs { get; set; } - - [JsonProperty("linkedServiceName")] - internal string LinkedServiceName { get; set; } - - [JsonProperty("typeProperties")] - internal DotNetActivityTypeProperties DotNetActivityTypeProperties { get; set; } - - - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityData.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityData.cs deleted file mode 100644 index 26cfb231..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityData.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class ActivityData - { - [JsonProperty("name")] - internal string Name { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityInput.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityInput.cs deleted file mode 100644 index 6375850d..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityInput.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class ActivityInput : ActivityData - { - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityOutput.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityOutput.cs deleted file mode 100644 index 23f48db2..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ActivityOutput.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class ActivityOutput : ActivityData - { - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/AzureSqlDatabaseLinkedService .cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/AzureSqlDatabaseLinkedService .cs deleted file mode 100644 index 99e21843..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/AzureSqlDatabaseLinkedService .cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class AzureSqlDatabaseLinkedService : LinkedService - { - [JsonProperty("properties")] - internal StorageServiceProperties Properties { get; set; } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ComputeService.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ComputeService.cs deleted file mode 100644 index 1e66f1f1..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/ComputeService.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class ComputeService : LinkedService - { - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/DotNetActivityTypeProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/DotNetActivityTypeProperties.cs deleted file mode 100644 index 34fe96fc..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/DotNetActivityTypeProperties.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class DotNetActivityTypeProperties - { - [JsonProperty("extendedProperties")] - internal Dictionary ExtendedProperties { get; set; } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/LinkedService.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/LinkedService.cs deleted file mode 100644 index 55ae4910..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/LinkedService.cs +++ /dev/null @@ -1,20 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class LinkedService - { - [JsonProperty("name")] - internal string Name { get; set; } - - public override int GetHashCode() - { - return Name.GetHashCode(); - } - - public override bool Equals(object obj) - { - return Name.Equals((obj as LinkedService).Name); - } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Pipeline.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Pipeline.cs deleted file mode 100644 index 46a89697..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Pipeline.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class Pipeline - { - [JsonProperty("properties")] - internal PipelineProperties Properties { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/PipelineProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/PipelineProperties.cs deleted file mode 100644 index 0e38b8fd..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/PipelineProperties.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Newtonsoft.Json; -using System.Collections.Generic; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class PipelineProperties - { - [JsonProperty("activities")] - internal List Activities { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageService.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageService.cs deleted file mode 100644 index 510d99b4..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageService.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class StorageService : LinkedService - { - [JsonProperty("properties")] - internal StorageServiceProperties Properties { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceProperties.cs deleted file mode 100644 index f31bdc6a..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceProperties.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class StorageServiceProperties - { - [JsonProperty("typeProperties")] - internal StorageServiceTypeProperties TypeProperties { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceTypeProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceTypeProperties.cs deleted file mode 100644 index e94b0430..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/StorageServiceTypeProperties.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class StorageServiceTypeProperties - { - [JsonProperty("connectionString")] - internal string ConnectionString { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Table.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Table.cs deleted file mode 100644 index c4197f86..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/Table.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class Table - { - [JsonProperty("properties")] - internal TableProperties Properties { get; set; } - - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableProperties.cs deleted file mode 100644 index 4a229eaa..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableProperties.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class TableProperties - { - [JsonProperty("linkedServiceName")] - internal string LinkedServiceName { get; set; } - - [JsonProperty("type")] - internal string Type { get; set; } - - [JsonProperty("typeProperties")] - internal TableTypeProperties TypeProperties { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableTypeProperties.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableTypeProperties.cs deleted file mode 100644 index 1efc4e1c..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Models/TableTypeProperties.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Newtonsoft.Json; - -namespace Microsoft.ADF.DotNetActivityRunner.Models -{ - internal class TableTypeProperties - { - [JsonProperty("folderPath")] - internal string FolderPath { get; set; } - - [JsonProperty("fileName")] - internal string FileName { get; set; } - - [JsonProperty("tableName")] - internal string TableName { get; set; } - } -} \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Properties/AssemblyInfo.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Properties/AssemblyInfo.cs deleted file mode 100644 index 0947f7fd..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("DotNetActivityRunner")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("DotNetActivityRunner")] -[assembly: AssemblyCopyright("Copyright © 2016")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("7808bbfc-83dc-49d4-bdae-467d6044fda1")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Runner.cs b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Runner.cs deleted file mode 100644 index 74309eb6..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/Runner.cs +++ /dev/null @@ -1,290 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; -using Microsoft.Azure.Management.DataFactories.Common.Models; -using Microsoft.Azure.Management.DataFactories.Models; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.ADF.DotNetActivityRunner -{ - public class Runner - { - public static DotNetActivityContext DeserializeActivity(string pipelineFileName, string activityName, string configFile = null, string adfFilesPath = @"..\..\..\McioppDataFactory") - { - // Get Key Vault settings if secure publish is being used on the local machine - AdfFileHelper adfFileHelper = null; - string settingsFile = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "SecurePublishSettings.json"); - if (File.Exists(settingsFile)) - { - AppSettings settings = JsonConvert.DeserializeObject(File.ReadAllText(settingsFile)); - X509Certificate2 cert = KeyVaultResolver.FindCertificateByThumbprint(settings.KeyVaultCertThumbprint); - string suffix = settings.EnvironmentSettings.First().KeyVaultDnsSuffix; - suffix = string.IsNullOrEmpty(suffix) ? "vault.azure.net:443" : suffix; - KeyVaultResolver keyVaultResolver = new KeyVaultResolver(settings.EnvironmentSettings.First().KeyVaultName, suffix, settings.KeyVaultCertClientId, cert); - adfFileHelper = new AdfFileHelper(keyVaultResolver, new Logger()); - } - - adfFilesPath = Path.GetFullPath(adfFilesPath); - - var deploymentDict = new Dictionary>(); - if (!string.IsNullOrEmpty(configFile)) - { - // Get deployment config - string deploymentConfigPath = Path.Combine(adfFilesPath, configFile); - var deploymentConfigJson = File.ReadAllText(deploymentConfigPath); - var deploymentJObj = JObject.Parse(deploymentConfigJson); - deploymentDict = deploymentJObj.Properties() - .ToDictionary(x => x.Name, - y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString())); - } - - DotNetActivityContext context = new DotNetActivityContext - { - LinkedServices = new List(), - Datasets = new List(), - Activity = new Activity(), - Logger = new ActivityLogger() - }; - - string pipelinePath = Path.Combine(adfFilesPath, pipelineFileName); - string pipelineJson = File.ReadAllText(pipelinePath); - - string pipelineName = Path.GetFileNameWithoutExtension(pipelineFileName); - - // Update with values from delpoyment config if exists - if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(pipelineName)) - { - JObject pipelineJObject = JObject.Parse(pipelineJson); - - foreach (KeyValuePair pair in deploymentDict[pipelineName]) - { - JToken token = pipelineJObject.SelectToken(pair.Key); - token.Replace(pair.Value); - } - - pipelineJson = pipelineJObject.ToString(); - } - - // Search for Key Vault references in the pipeline and replace with their Key Vault equivalents if found - if (adfFileHelper != null) - { - pipelineJson = adfFileHelper.ResolveKeyVault(pipelineJson).Result; - } - - var dummyPipeline = JsonConvert.DeserializeObject(pipelineJson); - - Models.Activity dummyActivity; - try - { - dummyActivity = dummyPipeline.Properties.Activities.Single(x => x.Name == activityName); - } - catch (InvalidOperationException) - { - throw new Exception($"Activity {activityName} not found in {pipelinePath}."); - } - - context.Activity.Name = dummyActivity.Name; - - context.Activity.TypeProperties = new DotNetActivity(); - DotNetActivity dotNetActivity = (DotNetActivity)context.Activity.TypeProperties; - dotNetActivity.ExtendedProperties = dummyActivity.DotNetActivityTypeProperties.ExtendedProperties; - - // get the input and output tables - var dummyDatasets = new HashSet(); - dummyDatasets.UnionWith(dummyActivity.Inputs); - dummyDatasets.UnionWith(dummyActivity.Outputs); - - var dummyServices = new HashSet(); - - // init the data tables - foreach (var dummyDataset in dummyDatasets) - { - // parse the table json source - var dataPath = Path.Combine(adfFilesPath, dummyDataset.Name + ".json"); - var dataJson = File.ReadAllText(dataPath); - var dummyTable = JsonConvert.DeserializeObject(dataJson); - { - // initialize dataset properties - DatasetTypeProperties datasetProperties; - switch (dummyTable.Properties.Type) - { - case "AzureBlob": - // init the azure model - var blobDataset = new AzureBlobDataset - { - FolderPath = dummyTable.Properties.TypeProperties.FolderPath, - FileName = dummyTable.Properties.TypeProperties.FileName - }; - - datasetProperties = blobDataset; - break; - - case "AzureTable": - case "AzureSqlTable": - var tableDataset = new AzureTableDataset - { - TableName = dummyTable.Properties.TypeProperties.TableName - }; - - datasetProperties = tableDataset; - break; - - case "SqlServerTable": - var sqlTableDataset = new SqlServerTableDataset(dummyTable.Properties.TypeProperties.TableName); - - datasetProperties = sqlTableDataset; - break; - - default: - throw new Exception($"Unexpected Dataset.Type {dummyTable.Properties.Type}"); - } - - // initialize dataset - - var dataDataset = new Dataset( - dummyDataset.Name, - new DatasetProperties( - datasetProperties, - new Availability(), - string.Empty - ) - ); - - dataDataset.Properties.LinkedServiceName = dummyTable.Properties.LinkedServiceName; - context.Datasets.Add(dataDataset); - - } - - // register the inputs and outputs in the activity - if (dummyDataset is Models.ActivityInput) - { - context.Activity.Inputs.Add(new ActivityInput(dummyDataset.Name)); - } - - if (dummyDataset is Models.ActivityOutput) - { - context.Activity.Outputs.Add(new ActivityOutput(dummyDataset.Name)); - } - - // parse the linked service json source for later use - string linkedServiceName = dummyTable.Properties.LinkedServiceName; - var servicePath = Path.Combine(adfFilesPath, linkedServiceName + ".json"); - string serviceJson = File.ReadAllText(servicePath); - - string linkedServiceType = string.Empty; - - // Update with values from delpoyment config if exists - if (deploymentDict.Count > 0 && deploymentDict.ContainsKey(linkedServiceName)) - { - JObject serviceJObject = JObject.Parse(serviceJson); - linkedServiceType = serviceJObject["properties"]["type"].ToObject(); - - foreach (KeyValuePair pair in deploymentDict[linkedServiceName]) - { - JToken token = serviceJObject.SelectToken(pair.Key); - token.Replace(pair.Value); - } - - serviceJson = serviceJObject.ToString(); - } - else - { - JObject serviceJObject = JObject.Parse(serviceJson); - linkedServiceType = serviceJObject["properties"]["type"].ToObject(); - } - - // Search for Key Vault references in the linked service and replace with their Key Vault equivalents if found - if (adfFileHelper != null) - { - serviceJson = adfFileHelper.ResolveKeyVault(serviceJson).Result; - } - - Models.LinkedService storageService; - - switch (linkedServiceType) - { - case "AzureSqlDatabase": - case "OnPremisesSqlServer": - storageService = JsonConvert.DeserializeObject(serviceJson); - break; - - case "AzureStorage": - storageService = JsonConvert.DeserializeObject(serviceJson); - break; - - default: - throw new Exception($"Mapper for linked service type '{linkedServiceType}' not found."); - } - - dummyServices.Add(storageService); - } - - // parse the hd insight service json source - var computeServicePath = Path.Combine(adfFilesPath, dummyActivity.LinkedServiceName + ".json"); - var computeServiceJson = File.ReadAllText(computeServicePath); - var computeService = JsonConvert.DeserializeObject(computeServiceJson); - - dummyServices.Add(computeService); - - - // init the services - foreach (var dummyService in dummyServices) - { - LinkedService linkedService = null; - - // init if it is a storage service - if (dummyService is Models.StorageService) - { - var dummyStorageService = dummyService as Models.StorageService; - - var service = new AzureStorageLinkedService - { - ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString - }; - - linkedService = new LinkedService( - dummyService.Name, - new LinkedServiceProperties(service) - ); - } - - // init if it is a AzureSqlDatabase service - if (dummyService is Models.AzureSqlDatabaseLinkedService) - { - var dummyStorageService = dummyService as Models.AzureSqlDatabaseLinkedService; - - var service = new AzureSqlDatabaseLinkedService() - { - ConnectionString = dummyStorageService.Properties.TypeProperties.ConnectionString - }; - - linkedService = new LinkedService( - dummyService.Name, - new LinkedServiceProperties(service) - ); - } - - // init if it is a hd insight service - if (dummyService is Models.ComputeService) - { - var service = new HDInsightLinkedService(); - linkedService = new LinkedService( - dummyService.Name, - new LinkedServiceProperties(service) - ); - } - - context.LinkedServices.Add(linkedService); - } - - return context; - } - } -} diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/app.config b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/app.config deleted file mode 100644 index 9a6f07a2..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/app.config +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/packages.config b/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/packages.config deleted file mode 100644 index 09737f25..00000000 --- a/SamplesV1/ADFCustomActivityRunner/DotNetActivityRunner/packages.config +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/NuGetPackage/ADFCustomActivityRunner.nuproj b/SamplesV1/ADFCustomActivityRunner/NuGetPackage/ADFCustomActivityRunner.nuproj deleted file mode 100644 index 49de086c..00000000 --- a/SamplesV1/ADFCustomActivityRunner/NuGetPackage/ADFCustomActivityRunner.nuproj +++ /dev/null @@ -1,45 +0,0 @@ - - - - - Debug - AnyCPU - - - Release - AnyCPU - - - - bb5b7de2-2a1a-4837-80c1-11c4693c28ec - - - $(MSBuildExtensionsPath)\NuProj\ - - - - ADFCustomActivityRunner - 1.0.4.4 - ADF Custom Activity Runner - daosul - daosul - Allows you to step into and debug Azure Data Factory (ADF) custom DotNetActivity activities using the information configured in your pipeline. - This package provides an attribute that you can apply to your DotNetActivity. You specify the location of the pipeline which your custom activity is run from and the name of the activity in the pipeline. The activity is then de-serialized into an object which you can use to easily debug your custom activity in the context of the pipeline it is found in. This package comes with a base class which you inherit instead of implementing IDotNetActivity. This abstract bass class is called CustomActivityBase, it implements IDotNetActivity and has a number of methods that simplify getting information from the ADF Json files. How to use: 1. Inherit from the abstract bass class called CustomActivityBase. 2. Implement the method RunActivity. This method calls the execute method internally. The arguments for this method are exposed as public properties on the base class: LinkedServices, Datasets, Activity, and Logger. 3. Add the CustomActivity attribute to the RunActivity method and specify the relative location from the custom activity project to the pipeline file and the name of the actviity you wish to target. Optionally the name of the deployment config file to use e.g. [CustomActivity(ActivityName = "KickOffSproc", PipelineLocation = @"..\DataFactoryApp\PipelineBlobSample.json", DeployConfig = "Dev.json")] 4. Run the custom activity as if it were a unit test. - - - - - - - Copyright © Microsoft - ADF Azure Data Factory DotNetActivity Custom Activities - true - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/NuGetPackage/Readme.txt b/SamplesV1/ADFCustomActivityRunner/NuGetPackage/Readme.txt deleted file mode 100644 index 910a78bf..00000000 --- a/SamplesV1/ADFCustomActivityRunner/NuGetPackage/Readme.txt +++ /dev/null @@ -1,20 +0,0 @@ -ADF custom activity runner allows you to step into and debug Azure Data Factory (ADF) custom DotNetActivity activities using the information configured in your pipeline. - -This package provides an attribute that you can apply to your DotNetActivity. You specify the location of the pipeline which your custom activity is run from and the name of the activity in the pipeline. The activity is then de-serialized into an object which you can use to easily debug your custom activity in the context of the pipeline it is found in. You can optionally specify a deployment configuration file if one is used. - -This package comes with a base class which you inherit instead of implementing IDotNetActivity. This abstract bass class is called CustomActivityBase, it implements IDotNetActivity and has a number of methods that simplify getting information from the ADF Json files. - -How to use: -1. Inherit from the abstract base class called CustomActivityBase. -2. Implement the method RunActivity. This method calls the execute method internally. The arguments for this method are exposed as public properties on the base class: - LinkedServices - Datasets - Activity - Logger -3. Add the CustomActivity attribute to the RunActivity method and specify the relative location from the custom activity project to the pipeline file and the name of the actviity you wish to target. Optionally the name of the deployment config file to use e.g.: -[CustomActivity(ActivityName = "KickOffSproc", PipelineLocation = @"..\DataFactoryApp\PipelineBlobSample.json", DeployConfig = "Dev.json")] -4. Run the custom activity as if it were a unit test. - -This also works with ADF Secure publish so you do not need to expose connection strings in the ADF linked service files. Instead they will be read in from Azure Key Vault. See https://github.com/Azure/Azure-DataFactory/tree/master/Samples/ADFSecurePublish for more information on this. - -This package uses code from NUnit and DotNetActivityRunner packages. \ No newline at end of file diff --git a/SamplesV1/ADFCustomActivityRunner/README.md b/SamplesV1/ADFCustomActivityRunner/README.md deleted file mode 100644 index e45b556c..00000000 --- a/SamplesV1/ADFCustomActivityRunner/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Introduction -This code contains source code and nuget project for the ADFCustomActivityRunner nuget. -ADF custom activity runner allows you to step into and debug Azure Data Factory (ADF) custom DotNetActivity activities using the information configured in your pipeline. - -This package provides an attribute that you can apply to your custom activity method (DotNetActivity) in oder to tell it that the method is a custom activity which can be debugged directly. Inside the attribute you specify the location of the pipeline which your custom activity is run from and the name of the activity in the pipeline. When you wish to debug the activity, this information is used to build the ADF objects (LinkedServices, Datasets, Activity, Logger) used in the custom activity. This allows you to debug the custom activity as if it were running inside Azure Data Factory. You can optionally specify a deployment configuration file if one is used. Also if you are using the ADF Secure Publish Visual Studio extension, it will work with that also so you do not need to expose secrets in your source code. One thing to note however, the first environment which is set in the Secure Publish user settings will be the one that is picked. This environment will have an associated key vault so if you wish to select a different key vault (which is associated with a different environment), make sure to push that environment to the top of the list. - -This package comes with a base class which you inherit instead of implementing IDotNetActivity. This abstract bass class is called CustomActivityBase, it implements IDotNetActivity and has a number of methods that simplify getting information from the ADF Json files, such as: -GetExtendedProperty(string name) -GetAllExtendedProperties() -GetInputSqlConnectionString() -GetOutputSqlConnectionString() -GetSqlConnectionString(string datasetName) -GetLinkedService(string name) -GetDataset(string name) -GetBlobFolderPath(string datasetName) -GetBlob(string datasetName) -DownLoadFile(string blobFileName) -DownLoadLatestFile() -UploadFile(string localFilePath) -GetLatestBlobFileName() -RemoveFilesFromBlob() - -# How to use: -1. Create a class library project where you will write your custom activity code and install the ADFCustomActivityRunner nuget package: https://www.nuget.org/packages/ADFCustomActivityRunner -2. Inherit from the abstract base class called CustomActivityBase. -3. Implement the method 'RunActivity'. This method calls the execute method internally. The arguments for this method are exposed as public properties on the base class: - LinkedServices - Datasets - Activity - Logger - Add your logic to this method. There are a number of helper methods available within the base class to make it easier to access linked services, datasets, extended properties and perform some basic blob operations such as uploading and downloading. -4. Add the CustomActivity attribute to the RunActivity method and specify the relative location from the custom activity project to the pipeline file and the name of the actviity you wish to target. Optionally the name of the deployment config file to use e.g.: -[CustomActivity(ActivityName = "KickOffSproc", PipelineLocation = @"..\DataFactoryApp\PipelineBlobSample.json", DeployConfig = "Dev.json")] -5. Run the custom activity as if it were a unit test. - -This package uses code from NUnit, DotNetActivityRunner, and ADFSecurePublish packages. diff --git a/SamplesV1/ADFSecurePublish/ADFSecurePublish.sln b/SamplesV1/ADFSecurePublish/ADFSecurePublish.sln deleted file mode 100644 index 7e41e269..00000000 --- a/SamplesV1/ADFSecurePublish/ADFSecurePublish.sln +++ /dev/null @@ -1,46 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 14 -VisualStudioVersion = 14.0.25420.1 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SecurePublishMenuCommand", "SecurePublishMenuCommand\SecurePublishMenuCommand.csproj", "{A33638E4-35F5-4F32-8631-933475D468EC}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AdfKeyVaultDeployment", "AdfKeyVaultDeployment\AdfKeyVaultDeployment.csproj", "{CD48A616-A9F8-4FDC-BC6E-A620846B5735}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SecurePublishForm", "SecurePublishForm\SecurePublishForm.csproj", "{FC61DA30-74AD-4FAA-9AA1-211CD99D96CA}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SecurePublishFormTests", "SecurePublishFormTests\SecurePublishFormTests.csproj", "{C66739A4-1196-4EF0-8EA7-3086F11BAD0A}" -EndProject -Project("{FF286327-C783-4F7A-AB73-9BCBAD0D4460}") = "NuGetPackage", "NuGetPackage\NuGetPackage.nuproj", "{72D0CE19-6B36-4A1F-80FB-62642ADF7459}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {A33638E4-35F5-4F32-8631-933475D468EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A33638E4-35F5-4F32-8631-933475D468EC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A33638E4-35F5-4F32-8631-933475D468EC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A33638E4-35F5-4F32-8631-933475D468EC}.Release|Any CPU.Build.0 = Release|Any CPU - {CD48A616-A9F8-4FDC-BC6E-A620846B5735}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CD48A616-A9F8-4FDC-BC6E-A620846B5735}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CD48A616-A9F8-4FDC-BC6E-A620846B5735}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CD48A616-A9F8-4FDC-BC6E-A620846B5735}.Release|Any CPU.Build.0 = Release|Any CPU - {FC61DA30-74AD-4FAA-9AA1-211CD99D96CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {FC61DA30-74AD-4FAA-9AA1-211CD99D96CA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {FC61DA30-74AD-4FAA-9AA1-211CD99D96CA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {FC61DA30-74AD-4FAA-9AA1-211CD99D96CA}.Release|Any CPU.Build.0 = Release|Any CPU - {C66739A4-1196-4EF0-8EA7-3086F11BAD0A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C66739A4-1196-4EF0-8EA7-3086F11BAD0A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C66739A4-1196-4EF0-8EA7-3086F11BAD0A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C66739A4-1196-4EF0-8EA7-3086F11BAD0A}.Release|Any CPU.Build.0 = Release|Any CPU - {72D0CE19-6B36-4A1F-80FB-62642ADF7459}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {72D0CE19-6B36-4A1F-80FB-62642ADF7459}.Debug|Any CPU.Build.0 = Debug|Any CPU - {72D0CE19-6B36-4A1F-80FB-62642ADF7459}.Release|Any CPU.ActiveCfg = Release|Any CPU - {72D0CE19-6B36-4A1F-80FB-62642ADF7459}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfBuild.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfBuild.cs deleted file mode 100644 index 1c54f0cd..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfBuild.cs +++ /dev/null @@ -1,109 +0,0 @@ -using Microsoft.Win32; -using System.Diagnostics; -using System.IO; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - /// - /// This class is used to perform a build on the selected ADF project. - /// - public class AdfBuild - { - private ILogger logger; - - public AdfBuild(ILogger logger) - { - this.logger = logger; - } - - /// - /// Builds the project from specified project path. - /// - /// The project path. - /// Type of the build. - /// The build configuration. - /// - /// True if the build is successful - /// - public async Task Build(string projPath, string buildType = "rebuild", string buildConfig = "Debug") - { - logger.Write($"Building '{projPath}'"); - - return await Task.Run(() => - { - string devenv = Path.Combine(GetVisualStudioInstalledPath(), "devenv.com"); - - ProcessStartInfo startInfo = new ProcessStartInfo - { - CreateNoWindow = true, - UseShellExecute = false, - RedirectStandardOutput = true, - FileName = devenv, - WindowStyle = ProcessWindowStyle.Hidden, - Arguments = - $"\"{projPath}\" /{buildType} {buildConfig} /project \"{projPath}\" /projectconfig {buildConfig}" - }; - - bool result; - - using (Process process = Process.Start(startInfo)) - { - string outputLine; - - while ((outputLine = process.StandardOutput.ReadLine()) != null) - { - logger.Write(outputLine); - } - - process.WaitForExit(); - - result = process.ExitCode == 0; - return result; - } - }); - } - - /// - /// Gets the visual studio installed path. - /// - internal string GetVisualStudioInstalledPath() - { - var visualStudioInstalledPath = string.Empty; - var visualStudioRegistryPath = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\WOW6432Node\Microsoft\VisualStudio\14.0"); - if (visualStudioRegistryPath != null) - { - visualStudioInstalledPath = visualStudioRegistryPath.GetValue("InstallDir", string.Empty) as string; - } - - if (string.IsNullOrEmpty(visualStudioInstalledPath) || !Directory.Exists(visualStudioInstalledPath)) - { - visualStudioRegistryPath = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\VisualStudio\14.0"); - if (visualStudioRegistryPath != null) - { - visualStudioInstalledPath = visualStudioRegistryPath.GetValue("InstallDir", string.Empty) as string; - } - } - - if (string.IsNullOrEmpty(visualStudioInstalledPath) || !Directory.Exists(visualStudioInstalledPath)) - { - visualStudioRegistryPath = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\WOW6432Node\Microsoft\VisualStudio\12.0"); - if (visualStudioRegistryPath != null) - { - visualStudioInstalledPath = visualStudioRegistryPath.GetValue("InstallDir", string.Empty) as string; - } - } - - if (string.IsNullOrEmpty(visualStudioInstalledPath) || !Directory.Exists(visualStudioInstalledPath)) - { - visualStudioRegistryPath = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\VisualStudio\12.0"); - if (visualStudioRegistryPath != null) - { - visualStudioInstalledPath = visualStudioRegistryPath.GetValue("InstallDir", string.Empty) as string; - } - } - - return visualStudioInstalledPath; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfDeploy.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfDeploy.cs deleted file mode 100644 index 4a60f716..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfDeploy.cs +++ /dev/null @@ -1,335 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Hyak.Common; -using Microsoft.Azure; -using Microsoft.Azure.Management.DataFactories; -using Microsoft.Azure.Management.DataFactories.Models; -using Microsoft.IdentityModel.Clients.ActiveDirectory; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; -using Newtonsoft.Json.Linq; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class AdfDeploy - { - private string resourceManagerEndpoint = "https://management.azure.com/"; - - private string resourceGroupName; - private string dataFactoryName; - private IAdfFileHelper adfFileHelper; - private ILogger logger; - private IBlobUtilities blob; - private SettingsContext settingsContext; - - public AdfDeploy(IAdfFileHelper adfFileHelper, ILogger logger, IBlobUtilities blob, SettingsContext settingsContext, string resourceGroupName, string dataFactoryName) - { - this.logger = logger; - this.blob = blob; - this.adfFileHelper = adfFileHelper; - this.resourceGroupName = resourceGroupName; - this.dataFactoryName = dataFactoryName; - this.settingsContext = settingsContext; - } - - /// - /// Deploys the specified ADF files and custom activity packages to Azure. - /// - /// The files to process. - /// The output folder which contains the files and custom activity zips. - /// The deployment configuration information. - public async Task Deploy(List filesToProcess, string outputFolder, DeployConfigInfo deployConfig = null) - { - bool result = true; - logger.Write(string.Empty); - logger.Write($"Getting all ADF resources to deploy to Azure from output folder '{outputFolder}'", "Black"); - - List> allFilesTasks = filesToProcess.Select(async x => await adfFileHelper.GetFileInfo(x, deployConfig)).ToList(); - List allFiles = new List(); - - foreach (var allFilesTask in allFilesTasks) - { - allFiles.Add(await allFilesTask); - } - - List validFiles = allFiles.Where(x => x.IsValid).ToList(); - - if (!validFiles.Any()) - { - logger.Write($"No valid ADF files found in '{outputFolder}'", "Red"); - return false; - } - - logger.Write($"{validFiles.Count} file{(validFiles.Count == 1 ? string.Empty : "s")} retreived"); - logger.Write(string.Empty); - logger.Write($"Begin deploying ADF resources to '{dataFactoryName}'", "Black"); - - // Log invalid files - List invalidFiles = allFiles.Where(x => !x.IsValid).ToList(); - - if (invalidFiles.Any()) - { - logger.Write("The following files found in the output folder will not be published:"); - - foreach (AdfFileInfo invalidFile in invalidFiles) - { - logger.Write(invalidFile.FileName); - } - } - - DataFactoryManagementClient client = GetDataFactoryManagementClient(); - - // Deploy Package Zips before deploying ADF JSON files - List pipelines = validFiles.Where(x => x.FileType == FileType.Pipeline).ToList(); - List packages = pipelines.SelectMany(x => x.CustomActivityPackages).Distinct().ToList(); - - if (packages.Any()) - { - result &= await DeployCustomActivities(packages, validFiles, outputFolder); - } - - List linkedServices = validFiles.Where(x => x.FileType == FileType.LinkedService).ToList(); - - if (linkedServices.Any()) - { - logger.Write(string.Empty); - logger.Write("Deploying LinkedServices", "Black"); - - // Deploy non batch linked services first - var linkedServiceTaskDict = new Dictionary>(); - - foreach (var linkedService in linkedServices.Where(x => !x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) - { - linkedServiceTaskDict.Add(linkedService.Name, - client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, - dataFactoryName, linkedService.Name, - new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(linkedService.FileContents))); - } - - foreach (var item in linkedServiceTaskDict) - { - try - { - LinkedServiceCreateOrUpdateResponse response = await item.Value; - - if (response.StatusCode == HttpStatusCode.OK) - { - logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); - } - else - { - logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); - result = false; - } - } - catch (Exception e) - { - logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); - logger.WriteError(e); - result = false; - } - } - - // Deploy batch linked services next - var batchLinkedServiceTaskDict = new Dictionary>(); - - foreach (var batchLinkedService in linkedServices.Where(x => x.SubType.Equals("AzureBatch", StringComparison.InvariantCultureIgnoreCase))) - { - batchLinkedServiceTaskDict.Add(batchLinkedService.Name, - client.LinkedServices.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, - dataFactoryName, batchLinkedService.Name, - new LinkedServiceCreateOrUpdateWithRawJsonContentParameters(batchLinkedService.FileContents))); - } - - foreach (var item in batchLinkedServiceTaskDict) - { - try - { - LinkedServiceCreateOrUpdateResponse response = await item.Value; - - if (response.StatusCode == HttpStatusCode.OK) - { - logger.Write($"Linked service '{response.LinkedService.Name}' uploaded successfully.", "Green"); - } - else - { - logger.Write($"Linked service '{response.LinkedService.Name}' did not upload successfully. Response status: {response.Status}", "Red"); - result = false; - } - } - catch (Exception e) - { - logger.Write($"Linked service '{item.Key}' did not upload successfully. Error: {e.Message}", "Red"); - logger.WriteError(e); - result = false; - } - } - } - - List tables = validFiles.Where(x => x.FileType == FileType.Table).ToList(); - - if (tables.Any()) - { - logger.Write(string.Empty); - logger.Write("Deploying tables", "Black"); - - // Deploy tables next - var tableTaskDict = new Dictionary>(); - foreach (AdfFileInfo adfJsonFile in tables) - { - try - { - Task tableTask = client.Datasets.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, - adfJsonFile.Name, new DatasetCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents)); - - tableTaskDict.Add(adfJsonFile.Name, tableTask); - } - catch (Exception e) - { - logger.Write($"An error occurred uploading table '{adfJsonFile.Name}': " + e.Message, "Red"); - logger.WriteError(e); - result = false; - } - } - - foreach (var task in tableTaskDict) - { - try - { - DatasetCreateOrUpdateResponse response = await task.Value; - - if (response.StatusCode == HttpStatusCode.OK) - { - logger.Write($"Table '{task.Key}' uploaded successfully.", "Green"); - } - else - { - logger.Write( - $"Table '{task.Key}' did not upload successfully. Response status: {response.Status}", - "Red"); - result = false; - } - } - catch (CloudException ex) - { - if (ex.Error.Code == "TableAvailabilityUpdateNotSupported") - { - logger.Write($"It looks like you are trying to change the availability for the Table '{task.Key}'. Currently this is not supported by ADF so as work around you should delete the dataset and related pipleline in the Data Factory '{dataFactoryName}' and run the publish again.", "Red"); - } - else - { - logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); - } - - logger.WriteError(ex); - result = false; - } - catch (Exception ex) - { - logger.WriteError(ex); - logger.Write($"Table '{task.Key}' did not upload successfully. An error occurred: " + ex.Message, "Red"); - result = false; - } - } - } - - if (pipelines.Any()) - { - logger.Write(string.Empty); - logger.Write("Deploying pipelines", "Black"); - - // Deploy pipelines last - var pipelineTaskDict = new Dictionary>(); - foreach (AdfFileInfo adfJsonFile in pipelines) - { - pipelineTaskDict.Add(adfJsonFile.Name, client.Pipelines.CreateOrUpdateWithRawJsonContentAsync(resourceGroupName, dataFactoryName, - adfJsonFile.Name, new PipelineCreateOrUpdateWithRawJsonContentParameters(adfJsonFile.FileContents))); - } - - foreach (var item in pipelineTaskDict) - { - try - { - PipelineCreateOrUpdateResponse response = await item.Value; - - if (response.StatusCode == HttpStatusCode.OK) - { - logger.Write($"Pipeline '{response.Pipeline.Name}' uploaded successfully.", "Green"); - } - else - { - logger.Write($"Pipeline '{response.Pipeline.Name}' did not upload successfully. Response status: {response.Status}", "Red"); - result = false; - } - } - catch (Exception e) - { - logger.WriteError(e); - logger.Write($"An error occurred uploading pipeline '{item.Key}': " + e.Message, "Red"); - result = false; - } - } - } - - return result; - } - - /// - /// Deploys the custom activities. - /// - /// The packages. - /// All files. - /// The output folder. - private async Task DeployCustomActivities(List packages, List allFiles, string outputFolder) - { - logger.Write(string.Empty); - logger.Write("Deploying custom activity packages to blob storage", "Black"); - - var tasks = new List>(); - - foreach (CustomActivityPackageInfo package in packages) - { - // Get connection string for blob account to upload to - JObject jObject = allFiles.FirstOrDefault(x => x.Name == package.PackageLinkedService)?.JObject; - - var parts = package.PackageFile.Split('/'); - - if (parts.Length != 2) - { - throw new Exception("packageFile should have only one '/' in it. Current packageFile value: " + package.PackageFile); - } - - string connectionString = jObject?.SelectToken("$.properties.typeProperties.connectionString").ToObject(); - string localFilePath = Path.Combine(outputFolder, parts[1]); - string blobFolder = parts[0]; - - tasks.Add(blob.UploadFile(localFilePath, connectionString, blobFolder)); - } - - var packageUploadResults = await Task.WhenAll(tasks); - - return packageUploadResults.All(x => x); - } - - /// - /// Gets the data factory management client. - /// - private DataFactoryManagementClient GetDataFactoryManagementClient() - { - TokenCloudCredentials aadTokenCredentials = - new TokenCloudCredentials(settingsContext.SubscriptionId, - AzureAccessUtilities.GetAuthorizationHeaderNoPopup(settingsContext)); - - Uri resourceManagerUri = new Uri(resourceManagerEndpoint); - - DataFactoryManagementClient client = new DataFactoryManagementClient(aadTokenCredentials, resourceManagerUri); - - return client; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfFileHelper.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfFileHelper.cs deleted file mode 100644 index 325ae915..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfFileHelper.cs +++ /dev/null @@ -1,309 +0,0 @@ -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Schema; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Reflection; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public enum FileType - { - LinkedService, - Table, - Pipeline, - Unknown - } - - /// - /// This class finds information on the ADF file types. It uses the ADF json schemas in order to determine which files they are. - /// It also performs resolutions of deployment settings if specified in a deployment config and Key Vault tokens from the chosen Key Vault - /// - /// - public class AdfFileHelper : IAdfFileHelper - { - const string linkedServiceSchema = "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.LinkedService.json"; - const string tableSchema = "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.Table.json"; - const string configSchema = "http://datafactories.schema.management.azure.com/vsschemas/V1/Microsoft.DataFactory.Config.json"; - - private JSchema jsonLinkedServiceSchema; - private JSchema jsonTableSchema; - private JSchema jsonPipelineSchema; - private JSchema jsonConfigSchema; - private IKeyVaultResolver keyVaultResolver; - private ILogger logger; - - public AdfFileHelper(IKeyVaultResolver keyVaultResolver, ILogger logger) - { - this.keyVaultResolver = keyVaultResolver; - this.logger = logger; - } - - public async Task GetSchemas(IHttpClient httpClient) - { - // If publishing a second time, no need to get the schemas again - if (jsonLinkedServiceSchema == null) - { - var tasks = new List - { - httpClient.GetAsync(linkedServiceSchema).ContinueWith(x => - { - jsonLinkedServiceSchema = JSchema.Parse(x.Result.Content.ReadAsStringAsync().Result, - new JSchemaUrlResolver()); - }), - httpClient.GetAsync(configSchema).ContinueWith(x => - { - jsonConfigSchema = JSchema.Parse(x.Result.Content.ReadAsStringAsync().Result, - new JSchemaUrlResolver()); - }) - }; - - await Task.WhenAll(tasks); - - // Get schema for pipelines - var assembly = Assembly.GetExecutingAssembly(); - var pipelineSchemaResource = "Microsoft.ADF.Deployment.AdfKeyVaultDeployment.EmbeddedResources.PipelineSchema.json"; - var tableSchemaResource = "Microsoft.ADF.Deployment.AdfKeyVaultDeployment.EmbeddedResources.TableSchema.json"; - - using (Stream stream = assembly.GetManifestResourceStream(pipelineSchemaResource)) - { - using (StreamReader reader = new StreamReader(stream)) - { - string schemaText = reader.ReadToEnd(); - jsonPipelineSchema = JSchema.Parse(schemaText, new JSchemaUrlResolver()); - } - } - - using (Stream stream = assembly.GetManifestResourceStream(tableSchemaResource)) - { - using (StreamReader reader = new StreamReader(stream)) - { - string schemaText = reader.ReadToEnd(); - jsonTableSchema = JSchema.Parse(schemaText, new JSchemaUrlResolver()); - } - } - - httpClient.Dispose(); - } - } - - /// - /// Gets the information on the deployment configuration file. - /// - /// The file path. - public DeployConfigInfo GetDeployConfigInfo(string filePath) - { - try - { - string fileContents = File.ReadAllText(filePath); - JObject jObject = JObject.Parse(fileContents); - - if (jObject.IsValid(jsonConfigSchema)) - { - var deploymentDict = jObject.Properties() - .ToDictionary(x => x.Name, - y => y.Value.ToDictionary(z => z["name"].ToString(), z => z["value"].ToString())); - - return new DeployConfigInfo - { - FilePath = filePath, - FileName = Path.GetFileName(filePath), - DeploymentDictionary = deploymentDict - }; - } - - return null; - } - catch - { - return null; - } - } - - /// - /// Gets the information on the ADF file. - /// - /// The file path. - /// The deploy configuration. - public async Task GetFileInfo(string filePath, DeployConfigInfo deployConfig = null) - { - string fileContents = File.ReadAllText(filePath); - string fileName = Path.GetFileNameWithoutExtension(filePath); - - // Initialize props to default values - AdfFileInfo fileInfo = new AdfFileInfo - { - FileType = FileType.Unknown, - SubType = string.Empty, - Name = string.Empty, - IsValid = false, - FileContents = fileContents, - FileName = fileName - }; - - JObject jObject = null; - - try - { - jObject = JObject.Parse(fileContents); - - JObject propertyJObject = jObject.GetValue("properties", StringComparison.OrdinalIgnoreCase) as JObject; - JToken nameJToken = jObject.GetValue("name", StringComparison.OrdinalIgnoreCase); - - if (propertyJObject == null || nameJToken == null) - { - logger.Write($"{fileInfo.FileName} is a not a valid ADF file."); - return fileInfo; - } - - fileInfo.Name = nameJToken.ToObject(); - - if (deployConfig != null) - { - // Update ADF files with deploymnet settings if they exist - jObject = ResolveDeploymentSettings(jObject, fileInfo.Name, deployConfig); - fileInfo.FileContents = jObject.ToString(); - } - - JToken typeJToken = propertyJObject.GetValue("type", StringComparison.OrdinalIgnoreCase); - if (typeJToken != null) - { - fileInfo.SubType = typeJToken.ToObject(); - } - - if (fileInfo.SubType == "CPSServiceBusProxy" || jObject.IsValid(jsonLinkedServiceSchema)) - { - fileInfo.FileType = FileType.LinkedService; - fileInfo.IsValid = true; - - logger.Write($"Retreived LinkedService: {fileInfo.FileName}"); - } - else if (jObject.IsValid(jsonTableSchema)) - { - fileInfo.FileType = FileType.Table; - fileInfo.IsValid = true; - - logger.Write($"Retreived Dataset: {fileInfo.FileName}"); - } - else if (jObject.IsValid(jsonPipelineSchema)) - { - fileInfo.FileType = FileType.Pipeline; - fileInfo.IsValid = true; - - logger.Write($"Retreived Pipeline: {fileInfo.FileName}"); - - // Get all custom activity packages if available - JArray activities = propertyJObject.GetValue("activities", StringComparison.InvariantCultureIgnoreCase) as JArray; - - if (activities != null) - { - fileInfo.CustomActivityPackages = new List(); - - foreach (JObject activity in activities) - { - JToken activityTypeJToken = activity.GetValue("type", StringComparison.OrdinalIgnoreCase); - - if (activityTypeJToken != null && activityTypeJToken.ToObject().Equals("DotNetActivity", StringComparison.CurrentCultureIgnoreCase)) - { - CustomActivityPackageInfo packageInfo = new CustomActivityPackageInfo(); - - packageInfo.PackageLinkedService = activity.SelectToken("$.typeProperties.packageLinkedService")?.ToObject(); - packageInfo.PackageFile = activity.SelectToken("$.typeProperties.packageFile")?.ToObject(); - - logger.Write($"Retreived Custom Activity package: {packageInfo.PackageFile}"); - - fileInfo.CustomActivityPackages.Add(packageInfo); - } - } - } - } - else - { - fileInfo.FileType = FileType.Unknown; - logger.Write($"{fileInfo.FileName} is a not a valid ADF file."); - } - } - catch (Exception e) - { - fileInfo.ErrorException = e; - logger.Write($"{fileInfo.FileName} is a not a valid ADF file. Error message: {e}"); - logger.WriteError(e); - } - - if (fileInfo.IsValid) - { - // Search for keyvault tokens and resolve them - string keyVaultResolvedContents = await ResolveKeyVault(fileInfo.FileContents); - - if (fileInfo.FileContents != keyVaultResolvedContents) - { - fileInfo.FileContents = keyVaultResolvedContents; - fileInfo.JObject = JObject.Parse(fileInfo.FileContents); - } - else - { - fileInfo.JObject = jObject; - } - } - - return fileInfo; - } - - /// - /// Resolves the deployment settings from the deployment config. - /// - private JObject ResolveDeploymentSettings(JObject jObject, string name, DeployConfigInfo deployConfig) - { - // Update with values from delpoyment config if exists - if (deployConfig.DeploymentDictionary.Count > 0 && deployConfig.DeploymentDictionary.ContainsKey(name)) - { - foreach (KeyValuePair pair in deployConfig.DeploymentDictionary[name]) - { - JToken token = jObject.SelectToken(pair.Key); - - if (token == null) - { - logger.Write( - $"The deployment configuration file '{deployConfig.FileName}' contains a substitution for '{name}' however the JPath '{pair.Key}' is not found in '{name}'."); - } - else - { - token.Replace(pair.Value); - } - } - } - - return jObject; - } - - /// - /// Resolves the key vault tokens with the actual values from Key Vault. - /// - /// The file contents. - public async Task ResolveKeyVault(string fileContents) - { - var matches = Regex.Matches(fileContents, "\\.*?)\\>"); - - foreach (Match match in matches) - { - string value = match.Groups["key"].Value.Trim(); - - logger.Write($"Found Key Vault token '{value}'. Resolving from Key Vault '{keyVaultResolver.KeyVaultName}'."); - - // Get keyvault value - var secretTask = await keyVaultResolver.GetSecret(value); - string secret = secretTask.Value; - - fileContents = fileContents.Replace($"", secret); - } - - return fileContents; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfKeyVaultDeployment.csproj b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfKeyVaultDeployment.csproj deleted file mode 100644 index 515be4fa..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AdfKeyVaultDeployment.csproj +++ /dev/null @@ -1,175 +0,0 @@ - - - - - Debug - AnyCPU - {CD48A616-A9F8-4FDC-BC6E-A620846B5735} - Library - Properties - Microsoft.ADF.Deployment.AdfKeyVaultDeployment - Microsoft.ADF.Deployment.AdfKeyVaultDeployment - v4.5.2 - 512 - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Hyak.Common.1.0.2\lib\net45\Hyak.Common.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.dll - True - - - ..\packages\Microsoft.Azure.Common.2.0.4\lib\net45\Microsoft.Azure.Common.NetFramework.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.1.0.0\lib\net45\Microsoft.Azure.KeyVault.dll - True - - - ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - True - - - ..\packages\Microsoft.Azure.Management.DataFactories.4.11.0\lib\net45\Microsoft.Azure.Management.DataFactories.dll - True - - - ..\packages\Microsoft.Data.Edm.5.7.0\lib\net40\Microsoft.Data.Edm.dll - True - - - ..\packages\Microsoft.Data.OData.5.7.0\lib\net40\Microsoft.Data.OData.dll - True - - - ..\packages\Microsoft.Data.Services.Client.5.7.0\lib\net40\Microsoft.Data.Services.Client.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - True - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.12.0\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.dll - True - - - ..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.Extensions.Desktop.dll - True - - - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - True - - - ..\packages\WindowsAzure.Storage.4.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll - True - - - ..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll - True - - - ..\packages\Newtonsoft.Json.Schema.2.0.7\lib\net45\Newtonsoft.Json.Schema.dll - True - - - - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Extensions.dll - True - - - ..\packages\Microsoft.Net.Http.2.2.22\lib\net45\System.Net.Http.Primitives.dll - True - - - - ..\packages\System.Spatial.5.7.0\lib\net40\System.Spatial.dll - True - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AzureAccessUtilities.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AzureAccessUtilities.cs deleted file mode 100644 index 3615e2c8..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/AzureAccessUtilities.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading.Tasks; -using Microsoft.IdentityModel.Clients.ActiveDirectory; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class AzureAccessUtilities - { - public static async Task> GetDataFactories(SettingsContext settingsContext) - { - var dataFactories = new List(); - - string uri = - $"https://management.azure.com/subscriptions/{settingsContext.SubscriptionId}/resources?$filter=resourceType%20eq%20'Microsoft.DataFactory%2Fdatafactories'&api-version=2014-04-01-preview"; - HttpResponseMessage response = await ExecuteArmRequest(settingsContext, HttpMethod.Get, uri); - - string responseText = response.Content.ReadAsStringAsync().Result; - - var jObject = JObject.Parse(responseText); - - var itemArray = jObject["value"] as JArray; - - if (itemArray != null) - { - foreach (JToken jToken in itemArray) - { - JObject item = jToken as JObject; - - string id = item?["id"].ToObject(); - string resourceGroup = string.Empty; - - if (!string.IsNullOrEmpty(id)) - { - resourceGroup = id.Split('/')[4]; - } - - DataFactoryInfo dfi = new DataFactoryInfo - { - SubscriptionId = settingsContext.SubscriptionId, - Location = item?["location"].ToObject(), - Name = item?["name"].ToObject(), - ResourceGroup = resourceGroup - }; - - dataFactories.Add(dfi); - } - } - - return dataFactories; - } - - public static async Task ExecuteArmRequest(SettingsContext settingsContext, HttpMethod httpMethod, string uri, object requestBody = null) - { - Uri baseUrl = new Uri(uri); - - string authorizationToken = GetAuthorizationHeaderNoPopup(settingsContext); - - HttpClient client = new HttpClient(); - HttpRequestMessage request = new HttpRequestMessage(httpMethod, baseUrl); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authorizationToken); - - if (requestBody != null) - { - request.Content = new StringContent(JsonConvert.SerializeObject(requestBody), Encoding.UTF8, "application/json"); - } - - HttpResponseMessage response = await client.SendAsync(request); - - return response; - } - - /// TokenCloudCredentials - /// Gets the authorization header without popup. - /// - public static string GetAuthorizationHeaderNoPopup(SettingsContext settingsContext) - { - var authority = new Uri(new Uri("https://login.windows.net"), settingsContext.ActiveDirectoryTenantId); - var context = new AuthenticationContext(authority.AbsoluteUri); - var credential = new ClientCredential(settingsContext.AdfClientId, settingsContext.AdfClientSecret); - - AuthenticationResult result = context.AcquireTokenAsync(settingsContext.WindowsManagementUri, credential).Result; - if (result != null) - { - return result.AccessToken; - } - - throw new InvalidOperationException("Failed to acquire token"); - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/BlobUtilities.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/BlobUtilities.cs deleted file mode 100644 index 3917a54f..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/BlobUtilities.cs +++ /dev/null @@ -1,89 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class BlobUtilities : IBlobUtilities - { - private ILogger logger; - - public BlobUtilities(ILogger logger) - { - this.logger = logger; - } - - /// - /// Uploads the file to blob storage. - /// - /// The local file path. - /// The connection string. - /// The folder path. - public async Task UploadFile(string localFilePath, string connectionString, string folderPath) - { - bool result = true; - - try - { - var blockBlob = GetCloudBlockBlob(Path.GetFileName(localFilePath), connectionString, folderPath); - await blockBlob.UploadFromFileAsync(localFilePath, FileMode.Open); - - logger.Write($"'{localFilePath}' uploaded to blob sucessfully", "Green"); - } - catch (Exception e) - { - logger.Write($"Failed to upload {localFilePath} to blob. Error: {e.Message}", "Red"); - logger.WriteError(e); - result = false; - } - - return result; - } - - /// - /// Gets the cloud block blob. - /// - /// Name of the file. - /// The connection string. - /// The folder path. - public CloudBlockBlob GetCloudBlockBlob(string fileName, string connectionString, string folderPath) - { - var container = GetCloudBlobContainer(connectionString, folderPath); - - //Create a new container, if it does not exist - container.CreateIfNotExists(); - - // Get the block blob - CloudBlockBlob blockBlob = container.GetBlockBlobReference(fileName); - - return blockBlob; - } - - /// - /// Gets the cloud blob container. - /// - /// The connection string. - /// The folder path. - private CloudBlobContainer GetCloudBlobContainer(string connectionString, string folderPath) - { - CloudStorageAccount inputStorageAccount; - try - { - inputStorageAccount = CloudStorageAccount.Parse(connectionString); - } - catch - { - throw new Exception("The connection string is not in the correct format."); - } - - CloudBlobClient blobClient = inputStorageAccount.CreateCloudBlobClient(); - CloudBlobContainer container = blobClient.GetContainerReference(folderPath); - return container; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/PipelineSchema.json b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/PipelineSchema.json deleted file mode 100644 index b04ef0f7..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/PipelineSchema.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "id": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.Pipeline.json", - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Microsoft.Azure.Management.DataFactories.Models.Pipeline", - "description": "A pipeline defines activities.", - "type": "object", - "required": [ - "name", - "properties" - ], - "properties": { - "name": { - "description": "Name of the pipeline.", - "type": "string", - "maxLength": 260, - "pattern": "^[A-Za-z0-9_][^<>*#.%&:\\\\+?/]*$" - }, - "properties": { - "description": "Pipeline properties.", - "$ref": "#/definitions/pipelineProperties" - }, - "$schema": { - "type": "string" - } - }, - "additionalProperties": true, - "definitions": { - "pipelineProperties": { - "title": "Microsoft.Azure.Management.DataFactories.Models.PipelineProperties", - "description": "Pipeline properties.", - "type": "object", - "required": [ - "activities" - ], - "properties": { - "activities": { - "description": "Activities that belong to the pipeline.", - "type": "array", - "items": { - } - } - }, - "additionalProperties": true - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/TableSchema.json b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/TableSchema.json deleted file mode 100644 index fe25160e..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/EmbeddedResources/TableSchema.json +++ /dev/null @@ -1,1112 +0,0 @@ -{ - "id": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.Table.json", - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Microsoft.Azure.Management.DataFactories.Models.Table", - "description": "A table defines the schema of the data as well as its storage.", - "type": "object", - "required": [ - "name", - "properties" - ], - "properties": { - "name": { - "description": "Name of the table.", - "type": "string", - "maxLength": 260, - "pattern": "^[A-Za-z0-9_][^<>*#.%&:\\\\+?/]*$" - }, - "properties": { - "description": "Table properties.", - "$ref": "#/definitions/tablePropertiesTypes" - }, - "$schema": { - "type": "string" - } - }, - "additionalProperties": true, - "definitions": { - "dataElement": { - "title": "Microsoft.Azure.Management.DataFactories.Models.DataElement", - "description": "Data element defines the semantics of each column of a table.", - "type": "object", - "properties": { - "name": { - "description": "Name of the data element.", - "type": "string" - }, - "description": { - "description": "Description of the data element.", - "type": "string" - }, - "culture": { - "description": "Culture of the data element.", - "type": "string" - }, - "format": { - "description": "Format of the data element.", - "type": "string" - }, - "type": { - "description": "Type of the data element.", - "type": "string", - "enum": [ - "String", - "Int16", - "Int32", - "Int64", - "Single", - "Double", - "Decimal", - "Guid", - "Boolean", - "Enum", - "Datetime", - "DateTimeOffset", - "Timespan", - "Byte[]" - ] - } - }, - "additionalProperties": false - }, - "partitionValue": { - "title": "Microsoft.Azure.Management.DataFactories.Models.PartitionValue", - "description": "The value of a partition.", - "type": "object", - "properties": { } - }, - "dateTimePartitionValue": { - "title": "Microsoft.Azure.Management.DataFactories.Models.DateTimePartitionValue", - "description": "The date/time value of a partition.", - "type": "object", - "properties": { - "date": { - "description": "Name of variable containing date.", - "type": "string" - }, - "format": { - "description": "Format string for the Date value.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "DateTime" - ] - } - } - }, - "partitionValueTypes": { - "type": "object", - "required": [ - "type" - ], - "allOf": [ - { - "$ref": "#/definitions/partitionValue" - }, - { - "oneOf": [ - { - "$ref": "#/definitions/dateTimePartitionValue" - } - ] - } - ] - }, - "partition": { - "title": "Microsoft.Azure.Management.DataFactories.Models.Partition", - "description": "The partition definition.", - "type": "object", - "properties": { - "name": { - "description": "Name of the partition.", - "type": "string" - }, - "value": { - "description": "Value of the partition.", - "$ref": "#/definitions/partitionValueTypes" - } - }, - "additionalProperties": false - }, - "storageFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.StorageFormat", - "description": "The format definition of a storage.", - "type": "object", - "properties": { - "serializer": { - "description": "Serializer.", - "type": "string" - }, - "deserializer": { - "description": "Deserializer.", - "type": "string" - } - } - }, - "textFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.TextFormat", - "description": "Text format.", - "type": "object", - "properties": { - "columnDelimiter": { - "description": "The column delimiter.", - "type": "string" - }, - "rowDelimiter": { - "description": "The row delimiter.", - "type": "string" - }, - "escapeChar": { - "description": "The escape character.", - "type": "string" - }, - "quoteChar": { - "description": "The quote character.", - "type": "string" - }, - "nullValue": { - "description": "The null value string.", - "type": "string" - }, - "encodingName": { - "description": "The code page name of the preferred encoding. If miss, the default value is “utf-8”, unless BOM denotes another Unicode encoding. Refer to the “Name” column of the table in the following link to set supported values: https://msdn.microsoft.com/library/system.text.encoding.aspx.", - "type": "string" - }, - "treatEmptyAsNull": { - "description": "Treat empty column values in the text file as null. The default value is true.", - "type": "boolean" - }, - "skipLineCount": { - "description": "The number of lines/rows to be skipped when parsing text files. The default value is 0.", - "type": "integer" - }, - "firstRowAsHeader": { - "description": "When used as input, treat the first row of data as headers. When used as output, write the headers into the output as the first row of data. The default value is false.", - "type": "boolean" - }, - "type": { - "type": "string", - "enum": [ - "TextFormat" - ] - } - } - }, - "storageFormatTypes": { - "type": "object", - "required": [ - "type" - ], - "allOf": [ - { - "$ref": "#/definitions/storageFormat" - }, - { - "oneOf": [ - { - "$ref": "#/definitions/textFormat" - }, - { - "$ref": "#/definitions/avroFormat" - }, - { - "$ref": "#/definitions/jsonFormat" - }, - { - "$ref": "#/definitions/orcFormat" - }, - { - "$ref": "#/definitions/parquetFormat" - } - ] - } - ] - }, - "jsonFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.JsonFormat", - "description": "The data stored in JSON format.", - "type": "object", - "properties": { - "filePattern": { - "description": "File pattern of JSON. To be more specific, the way of separating a collection of JSON objects. The default value is 'setOfObjects'. It is case-sensitive.", - "$ref": "#/definitions/jsonFormatFilePattern" - }, - "nestingSeparator": { - "description": "The character used to separate nesting levels. Default value is '.' (dot).", - "type": "string" - }, - "encodingName": { - "description": "The code page name of the preferred encoding. If not provided, the default value is 'utf-8', unless the byte order mark (BOM) denotes another Unicode encoding. The full list of supported values can be found in the 'Name' column of the table of encodings in the following reference: https://msdn.microsoft.com/library/system.text.encoding.aspx#Anchor_5.", - "type": "string" - }, - "type": { - "type": "string", - "enum": [ - "JsonFormat" - ] - } - } - }, - "jsonFormatFilePattern": { - "title": "Microsoft.Azure.Management.DataFactories.Models.JsonFormatFilePattern", - "description": "JSON format file pattern. A property of JsonFormat", - "type": "string", - "enum": [ - "setOfObjects", - "arrayOfObjects" - ] - }, - "avroFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AvroFormat", - "description": "Avro format.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AvroFormat" - ] - } - } - }, - "orcFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.OrcFormat", - "description": "Optimized row columnar (ORC) format.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "OrcFormat" - ] - } - } - }, - "parquetFormat": { - "title": "Microsoft.Azure.Management.DataFactories.Models.ParquetFormat", - "description": "Parquet format.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ParquetFormat" - ] - } - } - }, - "compressionTypes": { - "type": "object", - "required": [ - "type" - ], - "allOf": [ - { - "$ref": "#/definitions/compression" - }, - { - "oneOf": [ - { - "$ref": "#/definitions/bzip2Compression" - }, - { - "$ref": "#/definitions/deflateCompression" - }, - { - "$ref": "#/definitions/gzipCompression" - } - ] - } - ] - }, - "compression": { - "title": "Microsoft.Azure.Management.DataFactories.Models.Compression", - "description": "The compression method used on a table.", - "type": "object" - }, - "bzip2Compression": { - "title": "Microsoft.Azure.Management.DataFactories.Models.BZip2Compression", - "description": "The BZip2 compression method used on a table.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "BZip2" - ] - } - } - }, - "deflateCompression": { - "title": "Microsoft.Azure.Management.DataFactories.Models.DeflateCompression", - "description": "The Deflate compression method used on a table.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "Deflate" - ] - }, - "level": { - "description": "The Deflate compression method used on a table.", - "$ref": "#/definitions/compressionLevel" - } - } - }, - "gzipCompression": { - "title": "Microsoft.Azure.Management.DataFactories.Models.GZipCompression", - "description": "The GZip compression method used on a table.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "GZip" - ] - }, - "level": { - "description": "The GZip compression method used on a table.", - "$ref": "#/definitions/compressionLevel" - } - } - }, - "compressionLevel": { - "title": "Microsoft.Azure.Management.DataFactories.Models.CompressionLevel", - "description": "All available compression levels.", - "type": "string", - "enum": [ - "Optimal", - "Fastest" - ] - }, - "azureBlobDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AzureBlobLocation", - "description": "The Azure blob storage.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AzureBlob" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "properties": { - "folderPath": { - "description": "The path of the Azure blob storage.", - "type": "string" - }, - "tableRootLocation": { - "description": "The root of blob path.", - "type": "string" - }, - "fileName": { - "description": "The name of the Azure blob.", - "type": "string" - }, - "partitionedBy": { - "description": "The partitions to be used by the path.", - "type": "array", - "items": { - "$ref": "#/definitions/partition" - } - }, - "format": { - "description": "The format of the Azure blob storage.", - "$ref": "#/definitions/storageFormatTypes" - }, - "compression": { - "description": "The data compression method used for the blob storage.", - "$ref": "#/definitions/compressionTypes" - } - }, - "additionalProperties": false - } - } - }, - "azureTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AzureTableDataset", - "description": "The Azure table storage.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AzureTable" - ] - }, - "typeProperties": { - "required": [ - "tableName" - ], - "description": "Properties specific to this table type.", - "type": "object", - "properties": { - "tableName": { - "description": "The table name of the Azure table storage.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "azureSqlTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AzureSqlTableDataset", - "description": "The Azure SQL Server database.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AzureSqlTable" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "required": [ - "tableName" - ], - "properties": { - "tableName": { - "description": "The table name of the Azure SQL database.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "azureSqlDataWarehouseTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AzureSqlDataWarehouseTableDataset", - "description": "The Azure Synapse Analytics dataset.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AzureSqlDWTable" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "required": [ - "tableName" - ], - "properties": { - "tableName": { - "description": "The table name of the Azure Synapse Analytics dataset.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "documentDbCollectionDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.DocumentDbCollectionDataset", - "description": "Document Database Collection location.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "DocumentDbCollection" - ] - }, - "typeProperties": { - "properties": { - "collectionName": { - "description": "Document Database collection name.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "azureDataLakeStoreDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AzureDataLakeStoreDataset", - "description": "Azure Data Lake Store dataset.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AzureDataLakeStore" - ] - }, - "typeProperties": { - "description": "Properties specific to this data set type.", - "type": "object", - "required": [ - "folderPath" - ], - "properties": { - "folderPath": { - "description": "Path to the folder in the Azure Data Lake Store.", - "type": "string" - }, - "fileName": { - "description": "The name of the file in the Azure Data Lake Store.", - "type": "string" - }, - "format": { - "description": "The format of the Data Lake Store.", - "$ref": "#/definitions/storageFormatTypes" - }, - "compression": { - "description": "The data compression method used for the item(s) in the Azure Data Lake Store.", - "$ref": "#/definitions/compressionTypes" - }, - "partitionedBy": { - "description": "Specify a dynamic path and filename for time series data.", - "type": "array", - "items": { - "$ref": "#/definitions/partition" - } - } - }, - "additionalProperties": false - } - } - }, - "sqlServerTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.SqlServerTableDataset", - "description": "The on-premises SQL Server database.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "SqlServerTable" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "required": [ - "tableName" - ], - "properties": { - "tableName": { - "description": "The table name of the on-premises SQL database.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "fileShareDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.FileShareDataset", - "description": "An on-premises file system.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "FileShare" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "properties": { - "folderPath": { - "description": "The name of the file folder.", - "type": "string" - }, - "fileName": { - "description": "The name of the file.", - "type": "string" - }, - "partitionedBy": { - "description": "The partitions to be used by the path.", - "type": "array", - "items": { - "$ref": "#/definitions/partition" - } - }, - "format": { - "description": "The format of the file.", - "$ref": "#/definitions/storageFormatTypes" - }, - "fileFilter": { - "description": "Files sets filter by wildcard.", - "type": "string" - }, - "compression": { - "description": "The data compression method used on files.", - "$ref": "#/definitions/compressionTypes" - } - }, - "additionalProperties": false - } - } - }, - "relationalTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.RelationalTableDataset", - "description": "Relational Table location.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "RelationalTable" - ] - }, - "typeProperties": { - "properties": { - "tableName": { - "description": "The table name.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "onPremisesCassandraTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.OnPremisesCassandraTableDataset", - "description": "A table in a Cassandra database.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "CassandraTable" - ] - }, - "typeProperties": { - "properties": { - "tableName": { - "description": "The table name of the Cassandra database.", - "type": "string" - }, - "keyspace": { - "description": "The keyspace of the Cassandra database.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "mongoDbCollectionDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.MongoDbCollectionDataset", - "description": "A collection in a MongoDB database.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "MongoDbCollection" - ] - }, - "typeProperties": { - "required": [ - "collectionName" - ], - "properties": { - "collectionName": { - "description": "Name of the collection in the MongoDB database.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "oracleTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.OracleTableDataset", - "description": "The on-premises Oracle database.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "OracleTable" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "required": [ - "tableName" - ], - "properties": { - "tableName": { - "description": "The table name of the on-premises Oracle database.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "customDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.CustomDataset", - "description": "Custom location.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "CustomDataset" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object" - } - } - }, - "oDataResourceDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.ODataResourceDataset", - "description": "Open Data Protocol (OData) Resource Dataset.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "ODataResource" - ] - }, - "typeProperties": { - "description": "Properties specific to this dataset type.", - "type": "object", - "properties": { - "path": { - "description": "OData resource path.", - "type": "string" - } - }, - "additionalProperties": false - } - } - }, - "webTableDataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.WebTableDataset", - "description": "HTML table in a web page.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "WebTable" - ] - }, - "typeProperties": { - "description": "Properties specific to this dataset type.", - "type": "object", - "required": [ - "index" - ], - "properties": { - "index": { - "description": "The zero-based index of the table in web page.", - "type": "integer", - "minimum": 0 - }, - "path": { - "description": "The relative URL to the web page from the linked service URL.", - "type": "string" - }, - "partitionedBy": { - "description": "The partitions to be used by the path.", - "type": "array", - "items": { - "$ref": "#/definitions/partition" - } - } - }, - "additionalProperties": false - } - } - }, - "amazonS3Dataset": { - "title": "Microsoft.Azure.Management.DataFactories.Models.AmazonS3Dataset", - "description": "A single Amazon Simple Storage Service (S3) object or a set of S3 objects.", - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "AmazonS3" - ] - }, - "typeProperties": { - "description": "Properties specific to this table type.", - "type": "object", - "required": [ - "bucketName" - ], - "properties": { - "bucketName": { - "description": "The name of the Amazon S3 bucket.", - "type": "string" - }, - "key": { - "description": "The key of the Amazon S3 object.", - "type": "string" - }, - "prefix": { - "description": "The prefix filter for the S3 object(s) name.", - "type": "string" - }, - "version": { - "description": "The version for the S3 object.", - "type": "string" - }, - "format": { - "description": "The format of the Amazon S3 object(s).", - "$ref": "#/definitions/storageFormatTypes" - }, - "compression": { - "description": "The data compression method used for the Amazon S3 object(s).", - "$ref": "#/definitions/compressionTypes" - } - }, - "additionalProperties": false - } - } - }, - "availability": { - "title": "Microsoft.Azure.Management.DataFactories.Models.Availability", - "description": "The scheduler definition.", - "type": "object", - "required": [ - "frequency", - "interval" - ], - "properties": { - "frequency": { - "description": "Frequency in terms of minute, hour, day, etc.", - "type": "string", - "enum": [ - "Minute", - "Hour", - "Day", - "Week", - "Month" - ] - }, - "interval": { - "description": "The interval of running the scheduler.", - "type": "integer" - }, - "anchorDateTime": { - "description": "The date used as a reference point for calculating slice start and end dates.", - "type": "object" - }, - "offset": { - "description": "The offset relative to the anchor time.", - "type": "string", - "pattern": "((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))" - }, - "style": { - "description": "The scheduler style.", - "type": "string", - "enum": [ - "NotSpecified", - "StartOfInterval", - "EndOfInterval" - ] - } - }, - "additionalProperties": false - }, - "validationPolicy": { - "title": "Microsoft.Azure.Management.DataFactories.Models.ValidationPolicy", - "description": "Validation policy.", - "type": "object", - "properties": { - "minimumRows": { - "description": "Minimum rows.", - "type": "integer" - }, - "minimumSizeMB": { - "description": "Minimum size in MB.", - "type": "number" - }, - "validationPriorityOrder": { - "description": "Validation priority order. Choose from OldestFirst or NewestFirst.", - "type": "string", - "enum": [ - "OldestFirst", - "NewestFirst" - ] - } - }, - "additionalProperties": false - }, - "latencyPolicy": { - "title": "Microsoft.Azure.Management.DataFactories.Models.LatencyPolicy", - "description": "Latency policy.", - "type": "object", - "properties": { - "latencyLength": { - "description": "Latency length.", - "type": "string", - "pattern": "((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))" - } - }, - "additionalProperties": false - }, - "externalDataPolicy": { - "title": "Microsoft.Azure.Management.DataFactories.Models.ExternalDataPolicy", - "description": "External data policy.", - "type": "object", - "properties": { - "dataDelay": { - "description": "Data delay.", - "type": "string", - "pattern": "((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))" - }, - "retryInterval": { - "description": "Retry interval.", - "type": "string", - "pattern": "((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))" - }, - "retryTimeout": { - "description": "Specifies the timeout for the activity to run. If there is value specified, it takes the value of TimeSpan.FromMilliseconds(-1) which means indefinite timeout.", - "type": "string", - "pattern": "((\\d+)\\.)?(\\d\\d):(60|([0-5][0-9])):(60|([0-5][0-9]))" - }, - "maximumRetry": { - "description": "Maximum retry.", - "type": "integer" - } - }, - "additionalProperties": false - }, - "policy": { - "title": "Microsoft.Azure.Management.DataFactories.Models.Policy", - "description": "Policy of a table.", - "type": "object", - "properties": { - "validation": { - "description": "Validation policy.", - "$ref": "#/definitions/validationPolicy" - }, - "latency": { - "description": "Latency policy.", - "$ref": "#/definitions/latencyPolicy" - }, - "externalData": { - "description": "External data policy.", - "$ref": "#/definitions/externalDataPolicy" - } - }, - "additionalProperties": false - }, - "tableProperties": { - "title": "Microsoft.Azure.Management.DataFactories.Models.TableProperties", - "description": "Table properties.", - "type": "object", - "properties": { - "description": { - "description": "Table description.", - "type": "string" - }, - "structure": { - "description": "Columns that define the structure of the table.", - "type": "array", - "items": { - "$ref": "#/definitions/dataElement" - } - }, - "availability": { - "description": "Scheduler of the table.", - "$ref": "#/definitions/availability" - }, - "policy": { - "description": "Policy applied to the table.", - "$ref": "#/definitions/policy" - }, - "external": { - "description": "If set to true, the table is an external data set.", - "type": "boolean" - }, - "linkedServiceName": { - "description": "The referenced data linkedService name.", - "type": "string" - } - } - }, - "tablePropertiesTypes": { - "type": "object", - "required": [ - "availability", - "linkedServiceName", - "type", - "typeProperties" - ], - "allOf": [ - { - "$ref": "#/definitions/tableProperties" - }, - { - "oneOf": [ - { - "$ref": "#/definitions/azureBlobDataset" - }, - { - "$ref": "#/definitions/azureTableDataset" - }, - { - "$ref": "#/definitions/azureSqlTableDataset" - }, - { - "$ref": "#/definitions/azureSqlDataWarehouseTableDataset" - }, - { - "$ref": "#/definitions/documentDbCollectionDataset" - }, - { - "$ref": "#/definitions/sqlServerTableDataset" - }, - { - "$ref": "#/definitions/fileShareDataset" - }, - { - "$ref": "#/definitions/oracleTableDataset" - }, - { - "$ref": "#/definitions/relationalTableDataset" - }, - { - "$ref": "#/definitions/azureDataLakeStoreDataset" - }, - { - "$ref": "#/definitions/oDataResourceDataset" - }, - { - "$ref": "#/definitions/webTableDataset" - }, - { - "$ref": "#/definitions/onPremisesCassandraTableDataset" - }, - { - "$ref": "#/definitions/mongoDbCollectionDataset" - }, - { - "$ref": "#/definitions/amazonS3Dataset" - }, - { - "$ref": "#/definitions/customDataset" - } - ] - } - ] - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/HttpClientProxy.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/HttpClientProxy.cs deleted file mode 100644 index 952916a0..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/HttpClientProxy.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class HttpClientProxy : HttpClient, IHttpClient - { - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IAdfFileHelper.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IAdfFileHelper.cs deleted file mode 100644 index 5df27fd9..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IAdfFileHelper.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface IAdfFileHelper - { - DeployConfigInfo GetDeployConfigInfo(string filePath); - - Task GetFileInfo(string filePath, DeployConfigInfo deployConfig = null); - - Task GetSchemas(IHttpClient httpClient); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IBlobUtilities.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IBlobUtilities.cs deleted file mode 100644 index d502447e..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IBlobUtilities.cs +++ /dev/null @@ -1,13 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface IBlobUtilities - { - Task UploadFile(string localFilePath, string connectionString, string folderPath); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IHttpClient.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IHttpClient.cs deleted file mode 100644 index c2cf2a2a..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IHttpClient.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface IHttpClient - { - Task GetAsync(string requestUri); - - void Dispose(); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IKeyVaultResolver.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IKeyVaultResolver.cs deleted file mode 100644 index 828dc2ee..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/IKeyVaultResolver.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Azure.KeyVault; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface IKeyVaultResolver - { - string KeyVaultName { get; set; } - - Task GetSecret(string identifier); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ILogger.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ILogger.cs deleted file mode 100644 index 159bf205..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ILogger.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface ILogger - { - void Write(string format, params object[] args); - - void WriteError(Exception e); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ISettingsContextManager.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ISettingsContextManager.cs deleted file mode 100644 index 94e6ec24..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/ISettingsContextManager.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public interface ISettingsContextManager - { - SettingsContext GetSettingsContext(string environment); - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/KeyVaultResolver.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/KeyVaultResolver.cs deleted file mode 100644 index e1e7247c..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/KeyVaultResolver.cs +++ /dev/null @@ -1,156 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; -using Microsoft.Azure.KeyVault; -using Microsoft.IdentityModel.Clients.ActiveDirectory; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class KeyVaultResolver : IKeyVaultResolver - { - public string KeyVaultName { get; set; } - private string keyVaultDnsSuffix { get; set; } - - private string keyVaultClientId; - private string keyVaultSecret; - private KeyVaultClient client; - private X509Certificate2 keyVaultCert; - - /// - /// Initializes a new instance of the class using client secret. - /// - /// Name of the key vault. - /// DNS suffix of the key vault. - /// The key vault client identifier. - /// The key vault secret. - public KeyVaultResolver(string keyVaultName, string keyVaultDnsSuffix, string keyVaultClientId, string keyVaultSecret) - { - KeyVaultName = keyVaultName; - this.keyVaultDnsSuffix = keyVaultDnsSuffix; - this.keyVaultClientId = keyVaultClientId; - this.keyVaultSecret = keyVaultSecret; - - client = new KeyVaultClient(GetToken); - } - - /// - /// Initializes a new instance of the class using the associated certificated. - /// - /// Name of the key vault. - /// DNS suffix of the key vault. - /// The key vault client identifier. - /// The key vault cert. - public KeyVaultResolver(string keyVaultName, string keyVaultDnsSuffix, string keyVaultClientId, X509Certificate2 keyVaultCert) - { - KeyVaultName = keyVaultName; - this.keyVaultDnsSuffix = keyVaultDnsSuffix; - this.keyVaultClientId = keyVaultClientId; - this.keyVaultCert = keyVaultCert; - - client = new KeyVaultClient(GetTokenUsingCert); - } - - /// - /// Gets the secret from the Key Vault using the identifier. - /// - public async Task GetSecret(string identifier) - { - string secretIdentifier = $"https://{KeyVaultName}.{keyVaultDnsSuffix}/secrets/{identifier}"; - - try - { - return await client.GetSecretAsync(secretIdentifier); - } - catch (Exception ex) - { - throw new Exception($"Unable to retreive secret identifier '{secretIdentifier}' from Key Vault '{KeyVaultName}'. Error: {ex.Message}"); - } - } - - /// - /// Finds the certificate by thumbprint. - /// - /// The certificate thumbprint. - public static X509Certificate2 FindCertificateByThumbprint(string thumbPrint) - { - X509Certificate2 cert; - - X509Store myLocalStore = new X509Store(StoreName.My, StoreLocation.LocalMachine); - cert = FindCertFromStore(thumbPrint, myLocalStore); - - if (cert != null) - { - return cert; - } - - X509Store myCurrentUserStore = new X509Store(StoreName.My, StoreLocation.CurrentUser); - cert = FindCertFromStore(thumbPrint, myCurrentUserStore); - - if (cert != null) - { - return cert; - } - - X509Store caLocalStore = new X509Store(StoreName.CertificateAuthority, StoreLocation.LocalMachine); - cert = FindCertFromStore(thumbPrint, caLocalStore); - - return cert; - } - - private static X509Certificate2 FindCertFromStore(string thumbPrint, X509Store store) - { - try - { - store.Open(OpenFlags.ReadOnly); - X509Certificate2Collection col = store.Certificates.Find(X509FindType.FindByThumbprint, thumbPrint, false); - if (col.Count == 0) - { - return null; - } - - return col[0]; - } - finally - { - store.Close(); - } - } - - /// - /// Gets the token used to connect to Key Vault. - /// - private async Task GetToken(string authority, string resource, string scope) - { - var authContext = new AuthenticationContext(authority); - ClientCredential clientCred = new ClientCredential(keyVaultClientId, keyVaultSecret); - - // Note: An exception here can indicate that the local cert has become corrupted. Please first try and install it again. - AuthenticationResult result = await authContext.AcquireTokenAsync(resource, clientCred); - - if (result == null) - { - throw new InvalidOperationException("Failed to obtain the JWT token"); - } - - return result.AccessToken; - } - - /// - /// Gets the token using the cert. - /// - /// The authority. - /// The resource. - /// The scope. - public async Task GetTokenUsingCert(string authority, string resource, string scope) - { - var assertionCert = new ClientAssertionCertificate(keyVaultClientId, keyVaultCert); - - var context = new AuthenticationContext(authority, TokenCache.DefaultShared); - var result = await context.AcquireTokenAsync(resource, assertionCert); - return result.AccessToken; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Logger.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Logger.cs deleted file mode 100644 index 9bc54d43..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Logger.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class Logger : ILogger - { - public Logger() - { - Trace.AutoFlush = true; - } - - public void Write(string format, params object[] args) - { - Console.WriteLine(format, args); - Trace.WriteLine(string.Format(format, args) + "\r\n"); - } - - public void WriteError(Exception e) - { - Console.WriteLine(e); - Trace.WriteLine(e + "\r\n"); - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AdfFileInfo.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AdfFileInfo.cs deleted file mode 100644 index f898e68b..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AdfFileInfo.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System; -using System.Collections.Generic; -using Newtonsoft.Json.Linq; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - /// - /// Information on ADF Files such as pipelines, linked services and tables (datasets) - /// - public class AdfFileInfo - { - public string FileName { get; set; } - - public bool IsValid { get; set; } - - public string Name { get; set; } - - public string FileContents { get; set; } - - public FileType FileType { get; set; } - - public JObject JObject { get; set; } - - public string SubType { get; set; } - - public Exception ErrorException { get; set; } - - public List CustomActivityPackages { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AppSettings.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AppSettings.cs deleted file mode 100644 index 2ad2109d..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/AppSettings.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Generic; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public class AppSettings : IAppSettings - { - public List Subscriptions { get; set; } - - public string KeyVaultCertThumbprint { get; set; } - - public string KeyVaultCertClientId { get; set; } - - public List EnvironmentSettings { get; set; } - - public string AdfClientId { get; set; } - - public string AzureTenantId { get; set; } - - public string WindowsManagementUri { get; set; } - - public string[] FilesToExclude { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/CustomActivityPackageInfo.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/CustomActivityPackageInfo.cs deleted file mode 100644 index 55ed4840..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/CustomActivityPackageInfo.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - /// - /// Information on custom activities - /// - public class CustomActivityPackageInfo : IEquatable - { - public string PackageLinkedService { get; set; } - - public string PackageFile { get; set; } - - public bool Equals(CustomActivityPackageInfo other) - { - if (PackageLinkedService == other?.PackageLinkedService && - PackageFile == other?.PackageFile) - { - return true; - } - - return false; - } - - public override int GetHashCode() - { - int hashPackageLinkedService = PackageLinkedService == null ? 0 : PackageLinkedService.GetHashCode(); - int hashPackageFile = PackageFile == null ? 0 : PackageFile.GetHashCode(); - - return hashPackageLinkedService ^ hashPackageFile; - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DataFactoryInfo.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DataFactoryInfo.cs deleted file mode 100644 index 8459b435..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DataFactoryInfo.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public class DataFactoryInfo - { - public string Name { get; set; } - public string SubscriptionId { get; set; } - public string ResourceGroup { get; set; } - - public string Location { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DeployConfigInfo.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DeployConfigInfo.cs deleted file mode 100644 index 8b34d0cf..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/DeployConfigInfo.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.Collections.Generic; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - /// - /// Information on deplyment configuration fies - /// - public class DeployConfigInfo - { - public string FilePath { get; set; } - - public string FileName { get; set; } - - public Dictionary> DeploymentDictionary { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/EnvironmentSettings.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/EnvironmentSettings.cs deleted file mode 100644 index 14e246a6..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/EnvironmentSettings.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public class EnvironmentSettings - { - public string Name { get; set; } - - public string KeyVaultName { get; set; } - - public string KeyVaultDnsSuffix { get; set; } - - public string DeploymentConfigName { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/IAppSettings.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/IAppSettings.cs deleted file mode 100644 index 83fe5806..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/IAppSettings.cs +++ /dev/null @@ -1,27 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public interface IAppSettings - { - List Subscriptions { get; set; } - - string KeyVaultCertThumbprint { get; set; } - - string KeyVaultCertClientId { get; set; } - - List EnvironmentSettings { get; set; } - - string AdfClientId { get; set; } - - string AzureTenantId { get; set; } - - string WindowsManagementUri { get; set; } - - string[] FilesToExclude { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/SettingsContext.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/SettingsContext.cs deleted file mode 100644 index 5eda565b..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/SettingsContext.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public class SettingsContext - { - public string SubscriptionId { get; set; } - - public string ActiveDirectoryTenantId { get; set; } - - public string AdfClientId { get; set; } - - public string AdfClientSecret { get; set; } - - public string WindowsManagementUri { get; set; } - - public string KeyVaultName { get; set; } - - public string KeyVaultDnsSuffix { get; set; } - - public string KeyVaultCertificateThumbprint { get; set; } - - public string KeyVaultCertClientId { get; set; } - - public string KeyVaultCertClientSecret { get; set; } - - public string DeploymentConfigName { get; set; } - - public List FilesToExclude { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/Subscription.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/Subscription.cs deleted file mode 100644 index 49e71144..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Models/Subscription.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models -{ - public class Subscription - { - public string FriendlyName { get; set; } - public string Id { get; set; } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Properties/AssemblyInfo.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Properties/AssemblyInfo.cs deleted file mode 100644 index d4123691..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("AdfKeyVaultDeployment")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("AdfKeyVaultDeployment")] -[assembly: AssemblyCopyright("Copyright © 2016")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("cd48a616-a9f8-4fdc-bc6e-a620846b5735")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/PublishManager.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/PublishManager.cs deleted file mode 100644 index 913989d7..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/PublishManager.cs +++ /dev/null @@ -1,161 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - /// - /// Builds, resolves tokens from KeyVault, and deploys the ADF project - /// - public class PublishManager - { - private ILogger logger; - private IBlobUtilities blob; - private IAdfFileHelper adfFileHelper; - private IHttpClient httpClient; - private SettingsContext settingsContext; - - public PublishManager(IKeyVaultResolver keyVault, SettingsContext settingsContext, ILogger logger = null, HttpClientProxy httpClient = null) - { - - if (logger == null) - { - this.logger = new Logger(); - } - else - { - this.logger = logger; - } - - if (httpClient == null) - { - this.httpClient = new HttpClientProxy(); - } - else - { - this.httpClient = httpClient; - } - - - blob = new BlobUtilities(logger); - adfFileHelper = new AdfFileHelper(keyVault, logger); - - this.settingsContext = settingsContext; - } - - /// - /// Initializes a new instance of the class. This constructor is used for testing. - /// - /// The logger. - /// The key vault. - /// The BLOB. - /// The settings context manager. - public PublishManager(ILogger logger, IKeyVaultResolver keyVault, IBlobUtilities blob, ISettingsContextManager settingsContextManager) - { - this.logger = logger; - this.blob = blob; - adfFileHelper = new AdfFileHelper(keyVault, logger); - } - - /// - /// Performs all steps required for the secure publish including building, resolving deployment settings, resolving key vault, and deploying files and custom activities to Azure. - /// - public async Task BuildAndSecurePublish(string projectPath, DataFactoryInfo dataFactory) - { - string dataFactoryName = dataFactory.Name; - string dataFactoryResourceGroup = dataFactory.ResourceGroup; - - AdfBuild build = new AdfBuild(logger); - - Task buildTask = build.Build(projectPath); - Task schemaDownloadTask = adfFileHelper.GetSchemas(httpClient); - - bool buildResult = await buildTask; - await schemaDownloadTask; - - - if (buildResult) - { - // Only debug builds are currently supported by the ADF build process - string adfOutputFolder = Path.Combine(Path.GetDirectoryName(projectPath), @"bin\debug"); - - var publishResult = await PublishFromOutputFolder(adfOutputFolder, dataFactoryResourceGroup, dataFactoryName); - - if (publishResult) - { - logger.Write(string.Empty); - logger.Write("Publish complete", "Black"); - logger.Write(string.Empty); - } - } - else - { - logger.Write("Build failed", "Red"); - logger.Write(string.Empty); - } - } - - /// - /// Publishes to the specified data factory given a prebuilt output folder - /// - /// True if the publish succeeds othewise false - public async Task PublishFromOutputFolder(string adfOutputFolder, string dataFactoryResourceGroup, string dataFactoryName) - { - try - { - // Get schemas used for determining ADF files - await adfFileHelper.GetSchemas(httpClient); - - List filesToProcess = Directory.GetFiles(adfOutputFolder, "*.json", SearchOption.TopDirectoryOnly).ToList(); - - var lowerExcludeList = settingsContext.FilesToExclude == null || !settingsContext.FilesToExclude.Any() - ? new List() - : settingsContext.FilesToExclude.Select(x => x.ToLowerInvariant()); - - filesToProcess = filesToProcess.Where(x => - { - var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(x); - return fileNameWithoutExtension != null && - !lowerExcludeList.Contains(fileNameWithoutExtension.ToLowerInvariant()); - }).ToList(); - - string deployConfigPath = string.IsNullOrEmpty(settingsContext.DeploymentConfigName) - ? null - : filesToProcess.FirstOrDefault(x => - { - var fileName = Path.GetFileName(x); - return fileName != null && - fileName.Equals(Path.GetFileNameWithoutExtension(settingsContext.DeploymentConfigName) + ".json", - StringComparison.InvariantCultureIgnoreCase); - }); - - DeployConfigInfo deployConfig = null; - - logger.Write(string.Empty); - - if (!string.IsNullOrEmpty(deployConfigPath)) - { - logger.Write("Using deployment configuration file: " + deployConfigPath); - deployConfig = adfFileHelper.GetDeployConfigInfo(deployConfigPath); - } - else - { - logger.Write("No deployment configuration file found.", "Orange"); - } - - AdfDeploy adf = new AdfDeploy(adfFileHelper, logger, blob, settingsContext, dataFactoryResourceGroup, dataFactoryName); - - return await adf.Deploy(filesToProcess, adfOutputFolder, deployConfig); - } - catch (Exception e) - { - logger.Write($"Error: {e.Message}"); - logger.WriteError(e); - return false; - } - } - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/SettingsContextManager.cs b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/SettingsContextManager.cs deleted file mode 100644 index f6b4c6af..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/SettingsContextManager.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.ADF.Deployment.AdfKeyVaultDeployment.Models; - -namespace Microsoft.ADF.Deployment.AdfKeyVaultDeployment -{ - public class SettingsContextManager : ISettingsContextManager - { - private AppSettings settings; - - public SettingsContextManager() - { - } - - public SettingsContextManager(AppSettings settings) - { - this.settings = settings; - } - - public SettingsContext GetSettingsContext(string environment) - { - var environmentSetting = settings.EnvironmentSettings.First(x => x.Name == environment); - - SettingsContext settingsContext = new SettingsContext - { - AdfClientId = settings.AdfClientId, - KeyVaultCertClientId = settings.KeyVaultCertClientId, - KeyVaultCertificateThumbprint = settings.KeyVaultCertThumbprint, - ActiveDirectoryTenantId = settings.AzureTenantId, - WindowsManagementUri = settings.WindowsManagementUri, - KeyVaultName = environmentSetting.KeyVaultName, - KeyVaultDnsSuffix = string.IsNullOrEmpty(environmentSetting.KeyVaultDnsSuffix) ? "vault.azure.net:443" : environmentSetting.KeyVaultDnsSuffix, - DeploymentConfigName = environmentSetting.DeploymentConfigName - }; - - return settingsContext; - } - - } -} diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/app.config b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/app.config deleted file mode 100644 index 7fae7393..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/app.config +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/packages.config b/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/packages.config deleted file mode 100644 index 9353bbfd..00000000 --- a/SamplesV1/ADFSecurePublish/AdfKeyVaultDeployment/packages.config +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/NuGetPackage/NuGetPackage.nuproj b/SamplesV1/ADFSecurePublish/NuGetPackage/NuGetPackage.nuproj deleted file mode 100644 index 982111f0..00000000 --- a/SamplesV1/ADFSecurePublish/NuGetPackage/NuGetPackage.nuproj +++ /dev/null @@ -1,44 +0,0 @@ - - - - - Debug - AnyCPU - - - Release - AnyCPU - - - - 72d0ce19-6b36-4a1f-80fb-62642adf7459 - - - $(MSBuildExtensionsPath)\NuProj\ - - - - ADFSecurePublish - 1.0.1.1 - ADFSecurePublish - daosul - daosul - ADF Secure Publish - Contains core library for ADF secure publish. See https://github.com/Azure/Azure-DataFactory/tree/master/Samples/ADFSecurePublish for more information. - - - - - - - Copyright © daosul - ADF Secure Publish Azure Data Factory - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/NuGetPackage/Readme.txt b/SamplesV1/ADFSecurePublish/NuGetPackage/Readme.txt deleted file mode 100644 index b9074d62..00000000 --- a/SamplesV1/ADFSecurePublish/NuGetPackage/Readme.txt +++ /dev/null @@ -1 +0,0 @@ -Contains core library for ADF secure publish. See https://github.com/Azure/Azure-DataFactory/tree/master/Samples/ADFSecurePublish for more information. \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/README.md b/SamplesV1/ADFSecurePublish/README.md deleted file mode 100644 index 6553f122..00000000 --- a/SamplesV1/ADFSecurePublish/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Introduction -The Visual Studio Azure Data Factory project template is very useful for creating and maintaining ADF projects. The project template comes with a publish context menu item which allows you to publish the Data Factory directly to Azure. -Inside linked Services and deployment configuration files we specify sensitive data such as connection strings to data bases and blob storage keys. In order to use the in built publish command these secrets must be in plain text. -![Alt text](ReadmeResources/LinkedServiceWithSecret.PNG) -It is not advisable and often against security policy to commit this sensitive data to the code repository. -ADF Secure Publish allows these secrets to be stored in Key Vault and only references to them are specified in the linked services/ deployment configs. These references are resolved in-momory during deployment time and the ADF project is published to Azure. These files can then be committed to source repository without exposing any secrets. -![Alt text](ReadmeResources/LinkedServiceWithoutSecret.PNG) - -ADF secure publish adds a new context item to the ADF project template allowing you to publish your ADF project without risk of exposing confidental data. -![Alt text](ReadmeResources/ContextMenuItem.jpg?raw=true "Secure Publish context menu item") - - -# Getting Started -The ADF Secure publish Visual studio Extension can be installed from: VSExtension\SecurePublishMenuCommand.vsix - -# Prerequisites -Cert based authentication is used to access Key Vault so a certificate will need to be previously associated with your target Key Vault and any user using ADF Secure Publish will need to have this cert installed on their local machine. -Instructions for associating a certificate with KeyVault can be found here: https://azure.microsoft.com/en-us/documentation/articles/key-vault-use-from-web-application/. -The thumbprint of the certificate also needs to be configured in the user settings. Instructions on what user settings need to be configured is described below. - -Any subscription which you wish to deploy a Data Factory must be associated with an AAD client ID. This client ID will be configured with the Secure Publish user settings. Instructions on how to create a new AAD client ID and associate target subscriptions are described next. - -# How to create a new AAD client ID to associate with a subscription -1. Open up PowerShell -2. Log in to Azure by typing in the cmd: - Login-AzureRmAccount -3. Change to the subscription you wish to use by typing the cmd: - Select-AzureRmSubscription -SubscriptionId "<enter subscription ID here>" -4. Create a new AD application by entering the cmd: - $azureAdApplication = New-AzureRmADApplication -DisplayName "<Enter a name for your AD App here>" -HomePage "<Enter a URL here, it can be anything>" -IdentifierUris "<Enter a unique URL here, it can be anything>" -Password "<create a password and enter it here>" - Note: The password must be placed in each KeyVault which you are using and assign the identifier SecurePublishAdfClientSecret to it. - The corresponding Client ID needs to be assigned to the KeyVaultCertClientId user setting variable (retrieved in step 7). See section; Configuring Secure Publish settings for information on user settings. -5. Create an AD service principal for this application: - New-AzureRmADServicePrincipal -ApplicationId $azureAdApplication.ApplicationId -6. Add the service principal to the Data Factory Contributor role. - New-AzureRmRoleAssignment -RoleDefinitionName "Data Factory Contributor" -ServicePrincipalName $azureAdApplication.ApplicationId.Guid -7. Note the client ID by running the following cmd and selecting the value of ApplicationId: - $azureAdApplication.ApplicationId.Guid -8. Add the secret corresponding to the KeyVaultCertClientId variable (generated above in step 4) to the KeyVault and assign the identifier SecurePublishAdfClientSecret to it. - -# Configuring Secure Publish user settings -The settings required for Secure Publish can be configured by going to - -Tools -> Options -> Data Factory -> Secure Publish -![Alt text](ReadmeResources/UserSettings.png) - -Here is a description of each setting: -* ADF AAD Client ID - This is the client ID of an Azure Active Directory application which has been associated with the subscription which has data factories that we want to publish to. -* Azure Tenant ID - This is the azure tenant ID associated with your account. -* Environment Settings - The environment settings, consisting of; Environment Name, associated Key Vault, and associated deployment config (if one exists) -* Files to Exclude - The name of files which will be excluded from the deployment to Azure. -* KeyVault Certificate Client ID - This is the client ID of an Azure Active Directory application which has the certificate associated with it. -* KeyVault Certificate Thumbprint - This is the thumbprint of a certificate which has been registered with all Key Vaults which Secure Publish accesses. It also needs to be installed on the local machine in order to access these Key Vaults. -* Subscriptions - The subscriptions which target Data Factories reside. Enter a friendly name and ID for each target subscription. - -Most of these settings will only need to be set once, however if you wish to add a new subscription or a new environment then you will need to update these settings. - -# How to add a new Subscription -Go to -Tools -> Options -> Data Factory -> Secure Publish -> Click the ellipsis for Subscriptions -Click add to add a new subscription -![Alt text](ReadmeResources/NewSub.png) - -In the FriendlyName field, enter a name that you will use to select your subscription in the Subscription dropdown. -And enter the subscription ID in the Id field. Next time you launch secure publish you will see your subscription in the Subscription dropdown. - -# How to add a new environment -Each environment is tied to a Key Vault and optionally a deployment configuration file. You can configure a new environment by going to: -Tools -> Options -> Data Factory -> Secure Publish -> Click the ellipsis for Environment Settings -Click add to add a new environment: -![Alt text](ReadmeResources/NewEnv.png) - -In the Name field, add a name you will use to identify the environment. -In the KeyVaultName field, add the name of the Key Vault you wish to associate with the environment. -If you wish to associate a deployment config then enter the name of the file in the DeploymentConfig field. If you do not have a deployment config, you can leave this field blank. - -# How to reference Key Vault secrets in your ADF project -Once the Key Vault that you wish to target has been associated with an environment, you will add your secrets and reference them in your ADF files. -You can add and view secrets directly through the Azure Key Vault UI: https://ms.portal.azure.com/#blade/HubsExtension/Resources/resourceType/Microsoft.KeyVault%2Fvaults -Once your secrets are added, you reference them in your ADF files by using the following token: -"" where the identifier corresponds to the name of your Key Vault secret. -These tokens can be placed anywhere in your pipelines, tables, linked services or deployment configs and they will be resolved at deployment time. - -# How to do a Secure Publish -Right click on the ADF project in Solution Explorer to open the project context menu and click on the first menu item called Secrue Publish. This will open the following form: -![Alt text](ReadmeResources/Form.png) - -Select the subscription which your data factory resides in. Next select the environment you want to deploy to and the data factory. Hit publish to trigger the deployment and keep an eye on the output window to see the status of the deployment. -The deployment process performs the following tasks in sequence: -Builds the ADF project. -Validates all ADF json files and selects the deployment config if it exists. -Identifies any custom activity packages and uploads them to Azure. -Deploys Linked Services to the selected data factory, followed by tables and pipelines. - -The output window updates in real time, so when the deployment is complete, you will be notified. -![Alt text](ReadmeResources/PublishCompleted.png) - diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/ContextMenuItem.jpg b/SamplesV1/ADFSecurePublish/ReadmeResources/ContextMenuItem.jpg deleted file mode 100644 index 4d7d2379..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/ContextMenuItem.jpg and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/Form.png b/SamplesV1/ADFSecurePublish/ReadmeResources/Form.png deleted file mode 100644 index a0ccc569..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/Form.png and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithSecret.PNG b/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithSecret.PNG deleted file mode 100644 index cf6385f6..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithSecret.PNG and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithoutSecret.PNG b/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithoutSecret.PNG deleted file mode 100644 index ea61a0cc..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/LinkedServiceWithoutSecret.PNG and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/NewEnv.png b/SamplesV1/ADFSecurePublish/ReadmeResources/NewEnv.png deleted file mode 100644 index 167e6a98..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/NewEnv.png and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/NewSub.png b/SamplesV1/ADFSecurePublish/ReadmeResources/NewSub.png deleted file mode 100644 index e4178467..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/NewSub.png and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/PublishCompleted.png b/SamplesV1/ADFSecurePublish/ReadmeResources/PublishCompleted.png deleted file mode 100644 index 71ce3d88..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/PublishCompleted.png and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/ReadmeResources/UserSettings.png b/SamplesV1/ADFSecurePublish/ReadmeResources/UserSettings.png deleted file mode 100644 index a7a3a0a6..00000000 Binary files a/SamplesV1/ADFSecurePublish/ReadmeResources/UserSettings.png and /dev/null differ diff --git a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.config b/SamplesV1/ADFSecurePublish/SecurePublishForm/App.config deleted file mode 100644 index 979517ac..00000000 --- a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.config +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml b/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml deleted file mode 100644 index b38dd0c6..00000000 --- a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - diff --git a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml.cs b/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml.cs deleted file mode 100644 index a3cded51..00000000 --- a/SamplesV1/ADFSecurePublish/SecurePublishForm/App.xaml.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Data; -using System.Linq; -using System.Threading.Tasks; -using System.Windows; - -namespace SecurePublishForm -{ - /// - /// Interaction logic for App.xaml - /// - public partial class App : Application - { - } -} diff --git a/SamplesV1/ADFSecurePublish/SecurePublishForm/MainWindow.xaml b/SamplesV1/ADFSecurePublish/SecurePublishForm/MainWindow.xaml deleted file mode 100644 index ed9f6f69..00000000 --- a/SamplesV1/ADFSecurePublish/SecurePublishForm/MainWindow.xaml +++ /dev/null @@ -1,41 +0,0 @@ - - -