Decompose Crossgen, remove CleanPublishOutput, replace ExtractArchive with *FileExtractToDirectory (#3927)

* Eliminate CleanPublishOutput

* Decompose Crossgen Task

* WiP

* TarGzFileExtractToDirectory

* FixModeFlags --> CHMod

Also various eliminations of dead code

* Tasks cleanup

Move all tasks to .tasks file. There is little value in keepint them in each source file as they are already being used assumptively by files that happen to get executed later.

Also eliminating uses of <Exec> for DotNet invocations

* Move to BuildTools implementation of TarGzCreateFromDirectory

* Eliminate Command.cs and helpers

* Remove dead code

* Revert TarGz from BuildTools

Latest build tools package has not picked up the task, though it is checked in.

* Disable ChMod on Windows

* Windows bug fix

* PR Feedback

* Finish changing Chmod caps

build_projects/dotnet-cli-build/AddMetadataIsPE.cs

@@ -0,0 +1,65 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Reflection.PortableExecutable;
+using System.Collections.Generic;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class AddMetadataIsPE : Task
+    {
+        [Required]
+        public ITaskItem[] Items { get; set; }
+
+        [Output]
+        public ITaskItem[] ResultItems { get; set; }
+
+        public override bool Execute()
+        {
+            var resultItemsList = new List<ITaskItem>();
+            
+            foreach (var item in Items)
+            {
+                var resultItem = new TaskItem(item);
+                item.CopyMetadataTo(resultItem);
+                
+                if (File.Exists(resultItem.GetMetadata("FullPath")) &&
+                    HasMetadata(resultItem.GetMetadata("FullPath")))
+                {
+                    resultItem.SetMetadata("IsPE", "True");
+                }
+                else
+                {
+                    resultItem.SetMetadata("IsPE", "False");
+                }
+
+                resultItemsList.Add(resultItem);
+            }
+
+            ResultItems = resultItemsList.ToArray();
+
+            return true;
+        }
+
+        private static bool HasMetadata(string pathToFile)
+        {
+            try
+            {
+                using (var inStream = File.OpenRead(pathToFile))
+                {
+                    using (var peReader = new PEReader(inStream))
+                    {
+                        return peReader.HasMetadata;
+                    }
+                }
+            }
+            catch (BadImageFormatException) { }
+
+            return false;
+        }
+    }
+}

build_projects/dotnet-cli-build/ChangeEntryPointLibraryName.cs

@@ -4,8 +4,6 @@
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
 
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-
 namespace Microsoft.DotNet.Cli.Build
 {
     public class ChangeEntryPointLibraryName : Task

build_projects/dotnet-cli-build/Chmod.cs

@@ -0,0 +1,73 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class Chmod : ToolTask
+    {
+        [Required]
+        public string File { get; set; }
+
+        [Required]
+        public string Mode { get; set; }
+
+        public bool Recursive { get; set; }
+
+        protected override bool ValidateParameters()
+        {
+            base.ValidateParameters();
+
+            if (!System.IO.File.Exists(File))
+            {
+                Log.LogError($"File '{File} does not exist.");
+
+                return false;
+            }
+
+            return true;
+        }
+
+        protected override string ToolName
+        {
+            get { return "chmod"; }
+        }
+
+        protected override MessageImportance StandardOutputLoggingImportance
+        {
+            get { return MessageImportance.High; } // or else the output doesn't get logged by default
+        }
+
+        protected override string GenerateFullPathToTool()
+        {
+            return "chmod";
+        }
+
+        protected override string GenerateCommandLineCommands()
+        {
+            return $"{GetRecursive()} {GetMode()} {GetFilePath()}";
+        }
+
+        private string GetFilePath()
+        {
+            return File;
+        }
+
+        private string GetMode()
+        {
+            return Mode;
+        }
+
+        private string GetRecursive()
+        {
+            if(Recursive)
+            {
+                return "--recursive";
+            }
+
+            return null;
+        }
+    }
+}

build_projects/dotnet-cli-build/CleanPublishOutput.cs

@@ -1,32 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class CleanPublishOutput : Task
-    {
-        [Required]
-        public string Path { get; set; }
-
-        [Required]
-        public string Name { get; set; }
-
-        [Required]
-        public bool DeleteRuntimeConfigJson { get; set; }
-
-        [Required]
-        public bool DeleteDepsJson { get; set; }
-
-        public override bool Execute()
-        {
-            PublishMutationUtilties.CleanPublishOutput(Path, Name, DeleteRuntimeConfigJson, DeleteDepsJson);
-
-            return true;
-        }
-    }
-}

build_projects/dotnet-cli-build/CliDependencyVersions.cs

@@ -1,16 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class CliDependencyVersions
-    {
-        public static readonly string SharedFrameworkVersion = "1.0.0";
-        public static readonly string SharedHostVersion = "1.0.1";
-        public static readonly string HostFxrVersion = "1.0.1";
-
-        public static readonly string SharedFrameworkChannel = "preview";
-        public static readonly string SharedHostChannel = "preview";
-        public static readonly string HostFxrChannel = "preview";
-    }
-}

build_projects/dotnet-cli-build/Crossgen.cs

@@ -0,0 +1,134 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public sealed class Crossgen : ToolTask
+    {
+        [Required]
+        public string SourceAssembly { get;set; }
+
+        [Required]
+        public string DestinationPath { get; set; }
+
+        [Required]
+        public string JITPath { get; set; }
+
+        public string CrossgenPath { get; set; }
+
+        public bool ReadyToRun { get; set; }
+
+        public ITaskItem[] PlatformAssemblyPaths { get; set; }
+
+        private string TempOutputPath { get; set; }
+
+        protected override bool ValidateParameters()
+        {
+            base.ValidateParameters();
+
+            if (!File.Exists(SourceAssembly))
+            {
+                Log.LogError($"SourceAssembly '{SourceAssembly}' does not exist.");
+
+                return false;
+            }
+
+            return true;
+        }
+
+        public override bool Execute()
+        {
+            TempOutputPath = Path.GetTempFileName();
+
+            var toolResult = base.Execute();
+
+            if (toolResult)
+            {
+                File.Copy(TempOutputPath, DestinationPath, overwrite: true);
+            }
+
+            if (File.Exists(TempOutputPath))
+            {
+                File.Delete(TempOutputPath);
+            }
+
+            return toolResult;
+        }
+
+        protected override string ToolName
+        {
+            get { return "crossgen"; }
+        }
+
+        protected override MessageImportance StandardOutputLoggingImportance
+        {
+            get { return MessageImportance.High; } // or else the output doesn't get logged by default
+        }
+
+        protected override string GenerateFullPathToTool()
+        {
+            if (CrossgenPath != null)
+            {
+                return CrossgenPath;
+            }
+
+            return "crossgen";
+        }
+
+        protected override string GenerateCommandLineCommands()
+        {
+            return $"{GetReadyToRun()} {GetInPath()} {GetOutPath()} {GetPlatformAssemblyPaths()} {GetJitPath()}";
+        }
+
+        private string GetReadyToRun()
+        {
+            if (ReadyToRun)
+            {
+                return "-readytorun";
+            }
+
+            return null;
+        }
+
+        private string GetInPath()
+        {
+            return $"-in {SourceAssembly}";
+        }
+        
+        private string GetOutPath()
+        {
+            return $"-out {TempOutputPath}";
+        }
+
+        private string GetPlatformAssemblyPaths()
+        {
+            var platformAssemblyPaths = String.Empty;
+
+            if (PlatformAssemblyPaths != null)
+            {
+                foreach (var excludeTaskItem in PlatformAssemblyPaths)
+                {
+                    platformAssemblyPaths += $"{excludeTaskItem.ItemSpec}{Path.PathSeparator}";
+                }
+            }
+            
+            return $" -platform_assemblies_paths {platformAssemblyPaths.Trim(':')}";
+        }
+        
+        private string GetJitPath()
+        {
+            return $"-JITPath {JITPath}";
+        }
+
+        protected override void LogToolCommand(string message)
+        {
+            base.LogToolCommand($"{base.GetWorkingDirectory()}> {message}");
+        }
+    }
+}

build_projects/dotnet-cli-build/CrossgenDirectory.cs

@@ -1,34 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class CrossgenDirectory : Task
-    {
-        [Required]
-        public string CoreCLRVersion { get; set; }
-
-        [Required]
-        public string JitVersion { get; set; }
-
-        [Required]
-        public string SharedFrameworkNameVersionPath { get; set; }
-
-        [Required]
-        public string SdkOutputDirectory { get; set; }        
-
-        public override bool Execute()
-        {
-            var crossgenUtil = new Crossgen(CoreCLRVersion, JitVersion);
-
-            crossgenUtil.CrossgenDirectory(SharedFrameworkNameVersionPath, SdkOutputDirectory);
-
-            return true;
-        }
-    }
-}

build_projects/dotnet-cli-build/DotNetNew.cs

@@ -0,0 +1,18 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class DotNetNew : DotNetTool
+    {
+        protected override string Command
+        {
+            get { return "new"; }
+        }
+
+        protected override string Args
+        {
+            get { return $""; }
+        }
+    }
+}

build_projects/dotnet-cli-build/DotNetPublish.cs

@@ -0,0 +1,102 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class DotNetPublish : DotNetTool
+    {
+        protected override string Command
+        {
+            get { return "publish"; }
+        }
+
+        protected override string Args
+        {
+            get { return $"{GetProjectPath()} {GetConfiguration()} {GetFramework()} {GetNativeSubdirectory()} {GetBuildBasePath()} {GetOutput()} {GetVersionSuffix()}"; }
+        }
+
+        public string BuildBasePath { get; set; }
+
+        public string Configuration { get; set; }
+
+        public string Framework { get; set; }
+
+        public bool NativeSubDirectory { get; set; }
+
+        public string Output { get; set; }
+
+        public string ProjectPath { get; set; }
+
+        public string VersionSuffix { get; set; }
+
+        private string GetBuildBasePath()
+        {
+            if (!string.IsNullOrEmpty(BuildBasePath))
+            {
+                return $"--build-base-path {BuildBasePath}";
+            }
+
+            return null;
+        }
+
+        private string GetConfiguration()
+        {
+            if (!string.IsNullOrEmpty(Configuration))
+            {
+                return $"--configuration {Configuration}";
+            }
+
+            return null;
+        }
+
+        private string GetFramework()
+        {
+            if (!string.IsNullOrEmpty(Framework))
+            {
+                return $"--framework {Framework}";
+            }
+
+            return null;
+        }
+
+        private string GetNativeSubdirectory()
+        {
+            if (NativeSubDirectory)
+            {
+                return $"--native-subdirectory";
+            }
+
+            return null;
+        }
+
+        private string GetOutput()
+        {
+            if (!string.IsNullOrEmpty(Output))
+            {
+                return $"--output {Output}";
+            }
+
+            return null;
+        }
+
+        private string GetProjectPath()
+        {
+            if (!string.IsNullOrEmpty(ProjectPath))
+            {
+                return $"{ProjectPath}";
+            }
+
+            return null;
+        }
+
+        private string GetVersionSuffix()
+        {
+            if (!string.IsNullOrEmpty(VersionSuffix))
+            {
+                return $"--version-suffix {VersionSuffix}";
+            }
+
+            return null;
+        }
+    }
+}

build_projects/dotnet-cli-build/DotNetRestore.cs

@@ -12,11 +12,13 @@ namespace Microsoft.DotNet.Cli.Build
 
         protected override string Args
         {
-            get { return $"{GetVerbosity()} {GetFallbackSource()}"; }
+            get { return $"{GetVerbosity()} {GetFallbackSource()} {GetPackages()}"; }
         }
 
         public string FallbackSource { get; set; }
 
+        public string Packages { get; set; }
+
         public string Verbosity { get; set; }
 
         private string GetFallbackSource()
@@ -29,6 +31,16 @@ namespace Microsoft.DotNet.Cli.Build
             return null;
         }
 
+        private string GetPackages()
+        {
+            if (!string.IsNullOrEmpty(Packages))
+            {
+                return $"--packages {Packages}";
+            }
+
+            return null;
+        }
+
         private string GetVerbosity()
         {
             if (!string.IsNullOrEmpty(Verbosity))

build_projects/dotnet-cli-build/DotNetTool.cs

@@ -4,6 +4,8 @@
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
 
+using Microsoft.DotNet.Cli.Build.Framework;
+
 namespace Microsoft.DotNet.Cli.Build
 {
     public abstract class DotNetTool : ToolTask
@@ -20,7 +22,7 @@ namespace Microsoft.DotNet.Cli.Build
 
         protected override string ToolName
         {
-            get { return HostArtifactNames.DotnetHostBaseName; }
+            get { return $"dotnet{Constants.ExeSuffix}"; }
         }
 
         protected override MessageImportance StandardOutputLoggingImportance

build_projects/dotnet-cli-build/ExtractArchive.cs

@@ -1,38 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using System.IO.Compression;
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-using Microsoft.DotNet.Cli.Build.Framework;
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class ExtractArchive : Task
-    {
-        [Required]
-        public string InputFile { get; set; }
-
-        [Required]
-        public string DestinationDirectory { get; set; }
-
-        public override bool Execute()
-        {
-            FS.Mkdirp(DestinationDirectory);
-
-            Log.LogMessage($"Extracting Archive '{InputFile}' to '{DestinationDirectory}'");
-
-            if (CurrentPlatform.IsWindows)
-            {
-                ZipFile.ExtractToDirectory(InputFile, DestinationDirectory);
-            }
-            else
-            {
-                Exec("tar", "xf", InputFile, "-C", DestinationDirectory);
-            }
-
-            return true;
-        }
-    }
-}

build_projects/dotnet-cli-build/FixModeFlags.cs

@@ -1,23 +0,0 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using Microsoft.Build.Framework;
-using Microsoft.Build.Utilities;
-
-using static Microsoft.DotNet.Cli.Build.Framework.BuildHelpers;
-
-namespace Microsoft.DotNet.Cli.Build
-{
-    public class FixModeFlags : Task
-    {
-        [Required]
-        public string Dir { get; set; }
-
-        public override bool Execute()
-        {
-            FS.FixModeFlags(Dir);
-
-            return true;
-        }
-    }
-}

build_projects/dotnet-cli-build/GenerateBuildVersionInfo.cs

@@ -6,7 +6,7 @@ using Microsoft.Build.Utilities;
 
 namespace Microsoft.DotNet.Cli.Build
 {
-    public class GenerateBuildVersionInfo : Task
+    public class GenerateBuildVersionInfo : ToolTask
     {
         [Required]
         public string RepoRoot { get; set; }
@@ -20,9 +20,6 @@ namespace Microsoft.DotNet.Cli.Build
         [Output]
         public int VersionPatch { get; set; }
 
-        [Output]
-        public string CommitHash { get; set; }
-
         [Output]
         public string CommitCount { get; set; }
 
@@ -50,12 +47,13 @@ namespace Microsoft.DotNet.Cli.Build
         [Output]
         public string BranchName { get; set; }
 
+        private int _commitCount;
+
         public override bool Execute()
         {
-            var branchInfo = new BranchInfo(RepoRoot);
+            base.Execute();
 
-            var commitCount = GitUtils.GetCommitCount();
-            var commitHash = GitUtils.GetCommitHash();
+            var branchInfo = new BranchInfo(RepoRoot);
 
             var buildVersion = new BuildVersion()
             {
@@ -63,13 +61,12 @@ namespace Microsoft.DotNet.Cli.Build
                 Minor = int.Parse(branchInfo.Entries["MINOR_VERSION"]),
                 Patch = int.Parse(branchInfo.Entries["PATCH_VERSION"]),
                 ReleaseSuffix = branchInfo.Entries["RELEASE_SUFFIX"],
-                CommitCount = commitCount
+                CommitCount = _commitCount
             };
 
             VersionMajor = buildVersion.Major;
             VersionMinor = buildVersion.Minor;
             VersionPatch = buildVersion.Patch;
-            CommitHash = commitHash;
             CommitCount = buildVersion.CommitCountString;
             ReleaseSuffix = buildVersion.ReleaseSuffix;
             VersionSuffix = buildVersion.VersionSuffix;
@@ -82,5 +79,30 @@ namespace Microsoft.DotNet.Cli.Build
 
             return true;
         }
+
+        protected override string ToolName
+        {
+            get { return "git"; }
+        }
+
+        protected override MessageImportance StandardOutputLoggingImportance
+        {
+            get { return MessageImportance.High; } // or else the output doesn't get logged by default
+        }
+
+        protected override string GenerateFullPathToTool()
+        {
+            return "git";
+        }
+
+        protected override string GenerateCommandLineCommands()
+        {
+            return $"rev-list --count HEAD";
+        }
+
+        protected override void LogEventsFromTextOutput(string line, MessageImportance importance)
+        {
+            _commitCount = int.Parse(line);
+        }
     }
 }

build_projects/dotnet-cli-build/GetCommitHash.cs

@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.Cli.Build
+{
+    public class GetCommitHash : ToolTask
+    {
+        [Required]
+        public string RepoRoot { get; set; }
+
+        [Output]
+        public string CommitHash { get; set; }
+
+        protected override string ToolName
+        {
+            get { return "git"; }
+        }
+
+        protected override MessageImportance StandardOutputLoggingImportance
+        {
+            get { return MessageImportance.High; } // or else the output doesn't get logged by default
+        }
+
+        protected override string GenerateFullPathToTool()
+        {
+            return "git";
+        }
+
+        protected override string GenerateCommandLineCommands()
+        {
+            return $"rev-parse HEAD";
+        }
+
+        protected override void LogEventsFromTextOutput(string line, MessageImportance importance)
+        {
+            CommitHash = line;
+        }
+    }
+}

build_projects/dotnet-cli-build/TarGzFileExtractToDirectory.cs

@@ -0,0 +1,112 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.IO.Compression;
+
+namespace Microsoft.DotNet.Build.Tasks
+{
+    public sealed class TarGzFileExtractToDirectory : ToolTask
+    {
+        /// <summary>
+        /// The path to the archive to extract.
+        /// </summary>
+        [Required]
+        public string SourceArchive { get; set; }
+
+        /// <summary>
+        /// The path of the directory to which the archive should be extracted.
+        /// </summary>
+        [Required]
+        public string DestinationDirectory { get; set; }
+
+        /// <summary>
+        /// Indicates if the destination directory should be cleaned if it already exists.
+        /// </summary>
+        public bool OverwriteDestination { get; set; }
+
+        protected override bool ValidateParameters()
+        {
+            base.ValidateParameters();
+
+            var retVal = true;
+
+            if (Directory.Exists(DestinationDirectory))
+            {
+                if (OverwriteDestination == true)
+                {
+                    Log.LogMessage(MessageImportance.Low, "'{0}' already exists, trying to delete before unzipping...", DestinationDirectory);
+                    Directory.Delete(DestinationDirectory, recursive: true);
+                }
+                else
+                {
+                    Log.LogWarning("'{0}' already exists. Did you forget to set '{1}' to true?", DestinationDirectory, nameof(OverwriteDestination));
+
+                    retVal = false;
+                }
+            }
+
+            if (!File.Exists(SourceArchive))
+            {
+                Log.LogError($"SourceArchive '{SourceArchive} does not exist.");
+
+                retVal = false;
+            }
+
+            if (retVal)
+            {
+                Log.LogMessage($"Creating Directory {DestinationDirectory}");
+                Directory.CreateDirectory(DestinationDirectory);
+            }
+            
+            return retVal;
+        }
+
+        public override bool Execute()
+        {
+            bool retVal = base.Execute();
+
+            if (!retVal)
+            {
+                Log.LogMessage($"Deleting Directory {DestinationDirectory}");
+                Directory.Delete(DestinationDirectory);
+            }
+
+            return retVal;
+        }
+
+        protected override string ToolName
+        {
+            get { return "tar"; }
+        }
+
+        protected override MessageImportance StandardOutputLoggingImportance
+        {
+            get { return MessageImportance.High; } // or else the output doesn't get logged by default
+        }
+
+        protected override string GenerateFullPathToTool()
+        {
+            return "tar";
+        }
+
+        protected override string GenerateCommandLineCommands()
+        {
+            return $"xf {GetSourceArchive()} -C {GetDestinationDirectory()}";
+        }
+        
+        private string GetSourceArchive()
+        {
+            return SourceArchive;
+        }
+
+        private string GetDestinationDirectory()
+        {
+            return DestinationDirectory;
+        }
+    }
+}