Merge pull request #14069 from michaelnebel/csharp/nugetexe

C#: Download `nuget.exe` in the dependency manager (if not present).

Author: michaelnebel

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -1,13 +1,11 @@
-using Semmle.Util;
-using System;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
-using System.Xml;
-using System.Net.Http;
 using System.Diagnostics.CodeAnalysis;
-using System.Threading.Tasks;
+using System.IO;
 using System.Runtime.InteropServices;
+using System.Xml;
+using Semmle.Util;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -283,17 +281,8 @@ XmlDocument IBuildActions.LoadXml(string filename)
 
         public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s);
 
-        private static async Task DownloadFileAsync(string address, string filename)
-        {
-            using var httpClient = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, address);
-            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
-            using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
-            await contentStream.CopyToAsync(stream);
-        }
-
         public void DownloadFile(string address, string fileName) =>
-            DownloadFileAsync(address, fileName).Wait();
+            FileUtils.DownloadFile(address, fileName);
 
         public IDiagnosticsWriter CreateDiagnosticsWriter(string filename) => new DiagnosticsStream(filename);
 

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -1,6 +1,7 @@
-using Semmle.Util.Logging;
 using System.Collections.Generic;
 using System.Linq;
+using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -190,7 +191,7 @@ private static BuildScript DownloadNugetExe<TAutobuildOptions>(IAutobuilder<TAut
             })
             &
             BuildScript.DownloadFile(
-                "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe",
+                FileUtils.NugetExeUrl,
                 path,
                 e => builder.Log(Severity.Warning, $"Failed to download 'nuget.exe': {e.Message}"))
             &

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs

@@ -1,5 +1,4 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
@@ -14,70 +13,91 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     internal class NugetPackages
     {
+        private readonly string nugetExe;
+        private readonly ProgressMonitor progressMonitor;
+
+        /// <summary>
+        /// The list of package files.
+        /// </summary>
+        private readonly FileInfo[] packageFiles;
+
+        /// <summary>
+        /// The computed packages directory.
+        /// This will be in the Temp location
+        /// so as to not trample the source tree.
+        /// </summary>
+        private readonly TemporaryDirectory packageDirectory;
+
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
         public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, ProgressMonitor progressMonitor)
         {
-            SourceDirectory = sourceDir;
-            PackageDirectory = packageDirectory;
+            this.packageDirectory = packageDirectory;
             this.progressMonitor = progressMonitor;
 
-            // Expect nuget.exe to be in a `nuget` directory under the directory containing this exe.
-            var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
-            var directory = Path.GetDirectoryName(currentAssembly)
-                ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
-            nugetExe = Path.Combine(directory, "nuget", "nuget.exe");
-
-            if (!File.Exists(nugetExe))
-                throw new FileNotFoundException(string.Format("NuGet could not be found at {0}", nugetExe));
-
-            packages = new DirectoryInfo(SourceDirectory)
+            nugetExe = ResolveNugetExe(sourceDir);
+            packageFiles = new DirectoryInfo(sourceDir)
                 .EnumerateFiles("packages.config", SearchOption.AllDirectories)
                 .ToArray();
         }
 
-        // List of package files to download.
-        private readonly FileInfo[] packages;
-
-        /// <summary>
-        /// The list of package files.
-        /// </summary>
-        public IEnumerable<FileInfo> PackageFiles => packages;
-
         /// <summary>
-        /// Download the packages to the temp folder.
+        /// Tries to find the location of `nuget.exe` in the nuget directory under the directory
+        /// containing the executing assembly. If it can't be found, it is downloaded to the
+        /// `.nuget` directory under the source directory.
         /// </summary>
-        /// <param name="pm">The progress monitor used for reporting errors etc.</param>
-        public void InstallPackages()
+        /// <param name="sourceDir">The source directory.</param>
+        private string ResolveNugetExe(string sourceDir)
         {
-            foreach (var package in packages)
+            var currentAssembly = System.Reflection.Assembly.GetExecutingAssembly().Location;
+            var directory = Path.GetDirectoryName(currentAssembly)
+                ?? throw new FileNotFoundException($"Directory path '{currentAssembly}' of current assembly is null");
+
+            var nuget = Path.Combine(directory, "nuget", "nuget.exe");
+            if (File.Exists(nuget))
             {
-                RestoreNugetPackage(package.FullName);
+                progressMonitor.FoundNuGet(nuget);
+                return nuget;
+            }
+            else
+            {
+                progressMonitor.LogInfo($"Nuget.exe could not be found at {nuget}");
+                return DownloadNugetExe(sourceDir);
             }
         }
 
-        /// <summary>
-        /// The source directory used.
-        /// </summary>
-        public string SourceDirectory
+        private string DownloadNugetExe(string sourceDir)
         {
-            get;
-            private set;
-        }
+            var directory = Path.Combine(sourceDir, ".nuget");
+            var nuget = Path.Combine(directory, "nuget.exe");
 
-        /// <summary>
-        /// The computed packages directory.
-        /// This will be in the Temp location
-        /// so as to not trample the source tree.
-        /// </summary>
-        public TemporaryDirectory PackageDirectory { get; }
+            // Nuget.exe already exists in the .nuget directory.
+            if (File.Exists(nuget))
+            {
+                progressMonitor.FoundNuGet(nuget);
+                return nuget;
+            }
+
+            Directory.CreateDirectory(directory);
+            progressMonitor.LogInfo("Attempting to download nuget.exe");
+            try
+            {
+                FileUtils.DownloadFile(FileUtils.NugetExeUrl, nuget);
+                progressMonitor.LogInfo($"Downloaded nuget.exe to {nuget}");
+                return nuget;
+            }
+            catch
+            {
+                // Download failed.
+                throw new FileNotFoundException("Download of nuget.exe failed.");
+            }
+        }
 
         /// <summary>
         /// Restore all files in a specified package.
         /// </summary>
         /// <param name="package">The package file.</param>
-        /// <param name="pm">Where to log progress/errors.</param>
         private void RestoreNugetPackage(string package)
         {
             progressMonitor.NugetInstall(package);
@@ -92,12 +112,12 @@ private void RestoreNugetPackage(string package)
             if (Util.Win32.IsWindows())
             {
                 exe = nugetExe;
-                args = string.Format("install -OutputDirectory {0} {1}", PackageDirectory, package);
+                args = string.Format("install -OutputDirectory {0} {1}", packageDirectory, package);
             }
             else
             {
                 exe = "mono";
-                args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, PackageDirectory, package);
+                args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, packageDirectory, package);
             }
 
             var pi = new ProcessStartInfo(exe, args)
@@ -133,7 +153,15 @@ private void RestoreNugetPackage(string package)
             }
         }
 
-        private readonly string nugetExe;
-        private readonly ProgressMonitor progressMonitor;
+        /// <summary>
+        /// Download the packages to the temp folder.
+        /// </summary>
+        public void InstallPackages()
+        {
+            foreach (var package in packageFiles)
+            {
+                RestoreNugetPackage(package.FullName);
+            }
+        }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -88,6 +88,9 @@ public void CommandFailed(string exe, string arguments, int exitCode) =>
         public void MissingNuGet() =>
             LogError("Missing nuget.exe");
 
+        public void FoundNuGet(string path) =>
+            LogInfo($"Found nuget.exe at {path}");
+
         public void MissingDotNet() =>
             LogError("Missing dotnet CLI");
 

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -1,13 +1,17 @@
 using System;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Security.Cryptography;
 using System.Text;
+using System.Threading.Tasks;
 
 namespace Semmle.Util
 {
     public static class FileUtils
     {
+        public const string NugetExeUrl = "https://dist.nuget.org/win-x86-commandline/latest/nuget.exe";
+
         public static string ConvertToWindows(string path)
         {
             return path.Replace('/', '\\');
@@ -91,5 +95,20 @@ public static string ComputeFileHash(string filePath)
                 hex.AppendFormat("{0:x2}", b);
             return hex.ToString();
         }
+
+        private static async Task DownloadFileAsync(string address, string filename)
+        {
+            using var httpClient = new HttpClient();
+            using var request = new HttpRequestMessage(HttpMethod.Get, address);
+            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
+            using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
+            await contentStream.CopyToAsync(stream);
+        }
+
+        /// <summary>
+        /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
+        /// </summary>
+        public static void DownloadFile(string address, string fileName) =>
+           DownloadFileAsync(address, fileName).Wait();
     }
 }