Add PJ support to TestAssets (#5109)

* Add PJ support to TestAssets

* Cleaning the test assets is hard, so throwing instead

* better assets filter

* PR feedback

* PR Feedback

* Fix NullRef

* Move lock to surround action + file read/write

src/Microsoft.DotNet.Cli.Utils/PathUtility.cs

@@ -88,6 +88,18 @@ namespace Microsoft.DotNet.Tools.Common
             }
         }
 
+        /// <summary>
+        /// Returns childItem relative to directory, with Path.DirectorySeparatorChar as separator
+        /// </summary>
+        public static string GetRelativePath(DirectoryInfo directory, FileSystemInfo childItem)
+        {
+            var path1 = EnsureTrailingSlash(directory.FullName);
+
+            var path2 = childItem.FullName;
+
+            return GetRelativePath(path1, path2, Path.DirectorySeparatorChar, true);
+        }
+
         /// <summary>
         /// Returns path2 relative to path1, with Path.DirectorySeparatorChar as separator
         /// </summary>

src/Microsoft.DotNet.TestFramework/Extensions/DirectoryInfoExtensions.cs

@@ -21,5 +21,15 @@ namespace Microsoft.DotNet.TestFramework
         {
             return new FileInfo(Path.Combine(subject.FullName, fileName));
         }
+
+        public static void EnsureExistsAndEmpty(this DirectoryInfo subject)
+        {
+            if (subject.Exists)
+            {
+                subject.Delete(true);
+            }
+
+            subject.Create();
+        }
     }
 }

src/Microsoft.DotNet.TestFramework/TestAssetInfo.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.Cli.Utils;
 using Microsoft.DotNet.PlatformAbstractions;
@@ -29,6 +30,10 @@ namespace Microsoft.DotNet.TestFramework
 
         private readonly TestAssetInventoryFiles _inventoryFiles;
 
+        private readonly FileInfo _dotnetExeFile;
+
+        private readonly string _projectFilePattern;
+
         internal DirectoryInfo Root 
         {
             get
@@ -37,18 +42,37 @@ namespace Microsoft.DotNet.TestFramework
             }
         }
 
-        internal TestAssetInfo(DirectoryInfo root, string assetName)
+        internal TestAssetInfo(DirectoryInfo root, string assetName, FileInfo dotnetExeFile, string projectFilePattern)
         {
-            if (!root.Exists)
+            if (root == null)
             {
-                throw new DirectoryNotFoundException($"Directory not found at '{root}'");
+                throw new ArgumentNullException(nameof(root));
             }
 
-            _assetName = assetName;
+            if (string.IsNullOrWhiteSpace(assetName))
+            {
+                throw new ArgumentException("Argument cannot be null or whitespace", nameof(assetName));
+            }
+
+            if (dotnetExeFile == null)
+            {
+                throw new ArgumentNullException(nameof(dotnetExeFile));
+            }
+
+            if (string.IsNullOrWhiteSpace(projectFilePattern))
+            {
+                throw new ArgumentException("Argument cannot be null or whitespace", nameof(projectFilePattern));
+            }
 
             _root = root;
 
-            _dataDirectory = new DirectoryInfo(Path.Combine(_root.FullName, DataDirectoryName));
+            _assetName = assetName;
+
+            _dotnetExeFile = dotnetExeFile;
+
+            _projectFilePattern = projectFilePattern;
+
+            _dataDirectory = _root.GetDirectory(DataDirectoryName);
 
             _inventoryFiles = new TestAssetInventoryFiles(_dataDirectory);
 
@@ -56,37 +80,69 @@ namespace Microsoft.DotNet.TestFramework
             {
                 _dataDirectory
             };
-
-            if (!_dataDirectory.Exists)
-            {
-                _dataDirectory.Create();
-            }
         }
 
         public TestAssetInstance CreateInstance([CallerMemberName] string callingMethod = "", string identifier = "")
         {
-            var instancePath = GetTestDestinationDirectoryPath(callingMethod, identifier);
+            var instancePath = GetTestDestinationDirectory(callingMethod, identifier);
 
-            var testInstance = new TestAssetInstance(this, new DirectoryInfo(instancePath));
+            var testInstance = new TestAssetInstance(this, instancePath);
 
             return testInstance;
         }
 
-        private string GetTestDestinationDirectoryPath(string callingMethod, string identifier)
+        internal IEnumerable<FileInfo> GetSourceFiles()
+        {
+            ThrowIfTestAssetDoesNotExist();
+
+            ThrowIfAssetSourcesHaveChanged();
+            
+            return GetInventory(
+                _inventoryFiles.Source, 
+                null, 
+                () => {});
+        }
+
+        internal IEnumerable<FileInfo> GetRestoreFiles()
+        {
+            ThrowIfTestAssetDoesNotExist();
+
+            ThrowIfAssetSourcesHaveChanged();
+            
+            return GetInventory(
+                _inventoryFiles.Restore, 
+                GetSourceFiles, 
+                DoRestore);
+        }
+
+        internal IEnumerable<FileInfo> GetBuildFiles()
+        {
+            ThrowIfTestAssetDoesNotExist();
+
+            ThrowIfAssetSourcesHaveChanged();
+            
+            return GetInventory(
+                _inventoryFiles.Build,
+                () => GetRestoreFiles()
+                        .Concat(GetSourceFiles()),
+                DoBuild);
+        }
+
+        private DirectoryInfo GetTestDestinationDirectory(string callingMethod, string identifier)
         {
 #if NET451
             string baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
 #else
             string baseDirectory = AppContext.BaseDirectory;
 #endif
-            return Path.Combine(baseDirectory, callingMethod + identifier, _assetName);
+            return new DirectoryInfo(Path.Combine(baseDirectory, callingMethod + identifier, _assetName));
         }
 
-        private List<FileInfo> LoadInventory(FileInfo file)
+        private IEnumerable<FileInfo> LoadInventory(FileInfo file)
         {
             if (!file.Exists)
             {
-                return null;
+                return Enumerable.Empty<FileInfo>();
             }
 
             var inventory = new List<FileInfo>();
@@ -106,6 +162,11 @@ namespace Microsoft.DotNet.TestFramework
             FileUtility.ReplaceWithLock(
                 filePath =>
                 {
+                    if (!_dataDirectory.Exists)
+                    {
+                        _dataDirectory.Create();
+                    }
+
                     using (var stream =
                         new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None))
                     {
@@ -128,29 +189,6 @@ namespace Microsoft.DotNet.TestFramework
                         .Where(f => !FilesToExclude.Contains(f.Name));    
         }
 
-        internal IEnumerable<FileInfo> GetSourceFiles()
-        {
-            return GetInventory(_inventoryFiles.Source, null, () => {});
-        }
-
-        internal IEnumerable<FileInfo> GetRestoreFiles()
-        {
-            return GetInventory(_inventoryFiles.Restore, GetSourceFiles, DoRestore);
-        }
-
-        internal IEnumerable<FileInfo> GetBuildFiles()
-        {
-            return GetInventory(
-                _inventoryFiles.Build,
-                () =>
-                {
-                    var preInventory = new List<FileInfo>(GetRestoreFiles());
-                    preInventory.AddRange(GetSourceFiles());
-                    return preInventory;
-                },
-                DoBuild);
-        }
-
         private IEnumerable<FileInfo> GetInventory(
             FileInfo file,
             Func<IEnumerable<FileInfo>> beforeAction,
@@ -178,11 +216,26 @@ namespace Microsoft.DotNet.TestFramework
                 preInventory = beforeAction();
             }
 
-            action();
+            Task.Run(async () => await ConcurrencyUtilities.ExecuteWithFileLockedAsync<object>(
+                _dataDirectory.FullName, 
+                lockedToken =>
+                {
+                    if (file.Exists)
+                    {
+                        inventory = LoadInventory(file);
+                    }
+                    else
+                    {
+                        action();
 
-            inventory = GetFileList().Where(i => !preInventory.Select(p => p.FullName).Contains(i.FullName));
+                        inventory = GetFileList().Where(i => !preInventory.Select(p => p.FullName).Contains(i.FullName));
 
-            SaveInventory(file, inventory);
+                        SaveInventory(file, inventory);
+                    }
+
+                    return Task.FromResult(new Object());
+                },
+                CancellationToken.None)).Wait();
 
             return inventory;
         }
@@ -191,16 +244,16 @@ namespace Microsoft.DotNet.TestFramework
         {
             Console.WriteLine($"TestAsset Restore '{_assetName}'");
 
-            var projFiles = _root.GetFiles("*.csproj", SearchOption.AllDirectories);
+            var projFiles = _root.GetFiles(_projectFilePattern, SearchOption.AllDirectories);
 
             foreach (var projFile in projFiles)
             {
-                var restoreArgs = new string[] { "restore", projFile.FullName, "/p:SkipInvalidConfigurations=true" };
+                var restoreArgs = new string[] { "restore", projFile.FullName };
 
-                var commandResult = Command.Create(new PathCommandResolverPolicy(), "dotnet", restoreArgs)
-                                       .CaptureStdOut()
-                                       .CaptureStdErr()
-                                       .Execute();
+                var commandResult = Command.Create(_dotnetExeFile.FullName, restoreArgs)
+                                    .CaptureStdOut()
+                                    .CaptureStdErr()
+                                    .Execute();
 
                 int exitCode = commandResult.ExitCode;
 
@@ -223,11 +276,11 @@ namespace Microsoft.DotNet.TestFramework
 
             Console.WriteLine($"TestAsset Build '{_assetName}'");
 
-            var commandResult = Command.Create(new PathCommandResolverPolicy(), "dotnet", args) 
-                                       .WorkingDirectory(_root.FullName)
-                                       .CaptureStdOut()
-                                       .CaptureStdErr()
-                                       .Execute();
+            var commandResult = Command.Create(_dotnetExeFile.FullName, args) 
+                                    .WorkingDirectory(_root.FullName)
+                                    .CaptureStdOut()
+                                    .CaptureStdErr()
+                                    .Execute();
 
             int exitCode = commandResult.ExitCode;
 
@@ -242,5 +295,74 @@ namespace Microsoft.DotNet.TestFramework
                 throw new Exception(message);
             }
         }
+
+        private void ThrowIfAssetSourcesHaveChanged()
+        {
+            if (!_dataDirectory.Exists)
+            {
+                return;
+            }
+
+            var dataDirectoryFiles = _dataDirectory.GetFiles("*", SearchOption.AllDirectories);
+
+            if (!dataDirectoryFiles.Any())
+            {
+                return;
+            }
+
+            var trackedFiles = _inventoryFiles.AllInventoryFiles.SelectMany(f => LoadInventory(f));
+
+            var assetFiles = GetFileList();
+
+            var untrackedFiles = assetFiles.Where(a => !trackedFiles.Any(t => t.FullName.Equals(a.FullName)));
+
+            if (untrackedFiles.Any())
+            {
+                var message = $"TestAsset {_assetName} has untracked files. " +
+                    "Consider cleaning the asset and deleting its `.tam` directory to " + 
+                    "recreate tracking files.\n\n" +
+                    $".tam directory: {_dataDirectory.FullName}\n" +
+                    "Untracked Files: \n";
+
+                message += String.Join("\n", untrackedFiles.Select(f => $" - {f.FullName}\n"));
+
+                throw new Exception(message);
+            }
+
+            var earliestDataDirectoryTimestamp =
+                dataDirectoryFiles
+                    .OrderBy(f => f.LastWriteTime)
+                    .First()
+                    .LastWriteTime;
+
+            if (earliestDataDirectoryTimestamp == null)
+            {
+                return;
+            }
+
+            var updatedSourceFiles = LoadInventory(_inventoryFiles.Source)
+                .Where(f => f.LastWriteTime > earliestDataDirectoryTimestamp);
+
+            if (updatedSourceFiles.Any())
+            {
+                var message = $"TestAsset {_assetName} has updated files. " +
+                    "Consider cleaning the asset and deleting its `.tam` directory to " + 
+                    "recreate tracking files.\n\n" +
+                    $".tam directory: {_dataDirectory.FullName}\n" +
+                    "Updated Files: \n";
+
+                message += String.Join("\n", updatedSourceFiles.Select(f => $" - {f.FullName}\n"));
+
+                throw new GracefulException(message);
+            }
+        }
+
+        private void ThrowIfTestAssetDoesNotExist()
+        {
+            if (!_root.Exists)
+            { 
+                throw new DirectoryNotFoundException($"Directory not found at '{_root.FullName}'"); 
+            } 
+        }
     }
 }

src/Microsoft.DotNet.TestFramework/TestAssetInventoryFiles.cs

@@ -72,5 +72,18 @@ namespace Microsoft.DotNet.TestFramework
 
             Build = new FileInfo(Path.Combine(inventoryFileDirectory.FullName, "build.txt"));
         }
+
+        public IEnumerable<FileInfo> AllInventoryFiles
+        {
+            get
+            {
+                return new List<FileInfo>
+                {
+                    Source,
+                    Restore,
+                    Build
+                };
+            }
+        }
     }
 }

src/Microsoft.DotNet.TestFramework/TestAssetKinds.cs

@@ -16,5 +16,7 @@ namespace Microsoft.DotNet.TestFramework
         public static string DesktopTestProjects = "DesktopTestProjects";
         
         public static string TestProjects = "TestProjects";
+
+        public static string NonRestoredTestProjects = "NonRestoredTestProjects";
     }
 }

src/Microsoft.DotNet.TestFramework/TestAssets.cs

@@ -15,14 +15,51 @@ namespace Microsoft.DotNet.TestFramework
     {
         private DirectoryInfo _root;
 
-        public TestAssets(DirectoryInfo assetsRoot)
+        private FileInfo _dotnetCsprojExe;
+
+        private FileInfo _dotnetProjectJsonExe;
+
+        private const string ProjectJsonSearchPattern = "project.json";
+
+        private const string CsprojSearchPattern = "*.csproj";
+
+        public TestAssets(DirectoryInfo assetsRoot, FileInfo dotnetCsprojExe, FileInfo dotnetProjectJsonExe)
         {
+            if (assetsRoot == null)
+            {
+                throw new ArgumentNullException(nameof(assetsRoot));
+            }
+
+            if (dotnetCsprojExe == null)
+            {
+                throw new ArgumentNullException(nameof(dotnetCsprojExe));
+            }
+
+            if (dotnetProjectJsonExe == null)
+            {
+                throw new ArgumentNullException(nameof(dotnetProjectJsonExe));
+            }
+            
             if (!assetsRoot.Exists)
             {
                 throw new DirectoryNotFoundException($"Directory not found at '{assetsRoot}'");
             }
 
+            if (!dotnetCsprojExe.Exists)
+            {
+                throw new FileNotFoundException("Csproj dotnet executable must exist", dotnetCsprojExe.FullName);
+            }
+
+            if (!dotnetProjectJsonExe.Exists)
+            {
+                throw new FileNotFoundException("project.json dotnet executable must exist", dotnetProjectJsonExe.FullName);
+            }
+
             _root = assetsRoot;
+
+            _dotnetCsprojExe = dotnetCsprojExe;
+
+            _dotnetProjectJsonExe = dotnetProjectJsonExe;
         }
 
         public TestAssetInfo Get(string name)
@@ -34,7 +71,48 @@ namespace Microsoft.DotNet.TestFramework
         {
             var assetDirectory = new DirectoryInfo(Path.Combine(_root.FullName, kind, name));
 
-            return new TestAssetInfo(assetDirectory, name);
+            return new TestAssetInfo(
+                assetDirectory, 
+                name, 
+                _dotnetCsprojExe,
+                CsprojSearchPattern);
+        }
+
+        public TestAssetInfo GetProjectJson(string name)
+        {
+            return GetProjectJson(TestAssetKinds.TestProjects, name);
+        }
+
+        public TestAssetInfo GetProjectJson(string kind, string name)
+        {
+            var assetDirectory = new DirectoryInfo(Path.Combine(_root.FullName, kind, name));
+
+            return new TestAssetInfo(
+                assetDirectory, 
+                name, 
+                _dotnetProjectJsonExe,
+                ProjectJsonSearchPattern);
+        }
+
+        public DirectoryInfo CreateTestDirectory([CallerMemberName] string callingMethod = "", string identifier = "")
+        {
+            var testDestination = GetTestDestinationDirectoryPath("temp", callingMethod, identifier);
+
+            var testDirectory = new DirectoryInfo(testDestination);
+
+            testDirectory.EnsureExistsAndEmpty();
+
+            return testDirectory;
+        }
+
+        private string GetTestDestinationDirectoryPath(string testProjectName, string callingMethod, string identifier)
+        {
+#if NET451
+            string baseDirectory = AppDomain.CurrentDomain.BaseDirectory;
+#else
+            string baseDirectory = AppContext.BaseDirectory;
+#endif
+            return Path.Combine(baseDirectory, callingMethod + identifier, testProjectName);
         }
     }
 }

test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/DirectoryInfoAssertions.cs

@@ -42,7 +42,11 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
 
-        public AndConstraint<DirectoryInfoAssertions> HaveTextFile(string expectedFile, string expectedContents, string because = "", params object[] reasonArgs)
+        public AndConstraint<DirectoryInfoAssertions> HaveTextFile(
+            string expectedFile,
+            string expectedContents,
+            string because = "",
+            params object[] reasonArgs)
         {
             this.HaveFile(expectedFile, because, reasonArgs);
 
@@ -58,10 +62,15 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
 
-        public AndConstraint<DirectoryInfoAssertions> NotHaveFile(string expectedFile)
+        public AndConstraint<DirectoryInfoAssertions> NotHaveFile(
+            string expectedFile, 
+            string because = "", 
+            params object [] reasonArgs)
         {
             var file = _dirInfo.EnumerateFiles(expectedFile, SearchOption.TopDirectoryOnly).SingleOrDefault();
-            Execute.Assertion.ForCondition(file == null)
+            Execute.Assertion
+                .ForCondition(file == null)
+                .BecauseOf(because, reasonArgs)
                 .FailWith("File {0} should not be found in directory {1}.", expectedFile, _dirInfo.FullName);
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
@@ -76,7 +85,10 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
 
-        public AndConstraint<DirectoryInfoAssertions> HaveTextFiles(IDictionary<string, string> expectedFiles, string because = "", params object[] reasonArgs)
+        public AndConstraint<DirectoryInfoAssertions> HaveTextFiles(
+            IDictionary<string, string> expectedFiles, 
+            string because = "", 
+            params object[] reasonArgs)
         {
             foreach (var expectedFile in expectedFiles)
             {
@@ -86,7 +98,11 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
 
-        public AndConstraint<DirectoryInfoAssertions> HaveFilesMatching(string expectedFilesSearchPattern, SearchOption searchOption, string because = "", params object[] reasonArgs)
+        public AndConstraint<DirectoryInfoAssertions> HaveFilesMatching(
+            string expectedFilesSearchPattern, 
+            SearchOption searchOption, 
+            string because = "", 
+            params object[] reasonArgs)
         {
             var matchingFileExists = _dirInfo.EnumerateFiles(expectedFilesSearchPattern, searchOption).Any();
 
@@ -99,11 +115,14 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
             return new AndConstraint<DirectoryInfoAssertions>(this);
         }
 
-        public AndConstraint<DirectoryInfoAssertions> NotHaveFiles(IEnumerable<string> expectedFiles)
+        public AndConstraint<DirectoryInfoAssertions> NotHaveFiles(
+            IEnumerable<string> expectedFiles, 
+            string because = "", 
+            params object [] reasonArgs)
         {
             foreach (var expectedFile in expectedFiles)
             {
-                NotHaveFile(expectedFile);
+                NotHaveFile(expectedFile, because, reasonArgs);
             }
 
             return new AndConstraint<DirectoryInfoAssertions>(this);

test/Microsoft.DotNet.Tools.Tests.Utilities/Assertions/FileInfoExtensions.cs

@@ -25,5 +25,10 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
         {
             return new FileInfoNuGetLock(subject);
         }
+
+        public static string ReadAllText(this FileInfo subject)
+        {
+            return File.ReadAllText(subject.FullName);
+        }
     }
 }

test/Microsoft.DotNet.Tools.Tests.Utilities/TestBase.cs

@@ -53,7 +53,10 @@ namespace Microsoft.DotNet.Tools.Test.Utilities
                 {
                     var assetsRoot = Path.Combine(RepoRoot, "TestAssets");
 
-                    s_testAssets = new TestAssets(new DirectoryInfo(assetsRoot)); 
+                    s_testAssets = new TestAssets(
+                        new DirectoryInfo(assetsRoot),
+                        new FileInfo(new EnvironmentProvider().GetCommandPath("dotnet")),
+                        new FileInfo(new RepoDirectoriesProvider().PjDotnet)); 
                 }
 
                 return s_testAssets;