Retry when lock file is occupied by other process

src/Microsoft.DotNet.ProjectModel/Graph/LockFileReader.cs

@@ -17,7 +17,7 @@ namespace Microsoft.DotNet.ProjectModel.Graph
     {
         public static LockFile Read(string lockFilePath)
         {
-            using (var stream = new FileStream(lockFilePath, FileMode.Open, FileAccess.Read, FileShare.Read))
+            using (var stream = ResilientFileStreamOpener.OpenFile(lockFilePath))
             {
                 try
                 {
@@ -34,7 +34,7 @@ namespace Microsoft.DotNet.ProjectModel.Graph
             }
         }
 
-        internal static LockFile Read(string lockFilePath, Stream stream)
+        public static LockFile Read(string lockFilePath, Stream stream)
         {
             try
             {

src/Microsoft.DotNet.ProjectModel/Utilities/ResilientFileStreamOpener.cs

@@ -0,0 +1,45 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.IO;
+using System.Threading;
+
+namespace Microsoft.DotNet.ProjectModel.Utilities
+{
+    public class ResilientFileStreamOpener
+    {
+        public static FileStream OpenFile(string filepath)
+        {
+            return OpenFile(filepath, 0);
+        }
+       
+        public static FileStream OpenFile(string filepath, int retry)
+        {
+            if (retry < 0)
+            {
+                throw new ArgumentException("Retry can't be fewer than 0", nameof(retry));
+            }
+            
+            var count = 0;
+            while (true)
+            {
+                try 
+                {
+                    return new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                }
+                catch
+                {
+                    if (++count > retry)
+                    {
+                        throw;
+                    }
+                    else
+                    {
+                        Thread.Sleep(500);
+                    }
+                }
+            }
+        }
+    }
+}

src/Microsoft.DotNet.ProjectModel/WorkspaceContext.cs

@@ -7,6 +7,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using Microsoft.DotNet.ProjectModel.Graph;
+using Microsoft.DotNet.ProjectModel.Utilities;
 
 namespace Microsoft.DotNet.ProjectModel
 {
@@ -205,8 +206,23 @@ namespace Microsoft.DotNet.ProjectModel
                 else
                 {
                     currentEntry.FilePath = Path.Combine(projectDirectory, LockFile.FileName);
-                    currentEntry.Model = LockFileReader.Read(currentEntry.FilePath);
-                    currentEntry.UpdateLastWriteTimeUtc();
+
+                    using (var fs = ResilientFileStreamOpener.OpenFile(currentEntry.FilePath, retry: 2))
+                    {
+                        try
+                        {
+                            currentEntry.Model = LockFileReader.Read(currentEntry.FilePath, fs);
+                            currentEntry.UpdateLastWriteTimeUtc();
+                        }
+                        catch (FileFormatException ex)
+                        {
+                            throw ex.WithFilePath(currentEntry.FilePath);
+                        }
+                        catch (Exception ex)
+                        {
+                            throw FileFormatException.Create(ex, currentEntry.FilePath);
+                        }
+                    }
                 }
             }
 

test/dotnet-projectmodel-server.Tests/DthTests.cs

@@ -4,6 +4,9 @@
 using System;
 using System.IO;
 using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.ProjectModel.Graph;
 using Microsoft.DotNet.TestFramework;
 using Microsoft.DotNet.Tools.Test.Utilities;
 using Microsoft.Extensions.Logging;
@@ -18,7 +21,7 @@ namespace Microsoft.DotNet.ProjectModel.Server.Tests
     {
         private readonly TestAssetsManager _testAssetsManager;
         private readonly ILoggerFactory _loggerFactory;
-        
+
         public DthTests()
         {
             _loggerFactory = new LoggerFactory();
@@ -32,11 +35,11 @@ namespace Microsoft.DotNet.ProjectModel.Server.Tests
             {
                 _loggerFactory.AddConsole(LogLevel.Information);
             }
-            else
+            else if (testVerbose == "0")
             {
                 _loggerFactory.AddConsole(LogLevel.Warning);
             }
-            
+
             _testAssetsManager = new TestAssetsManager(
                 Path.Combine(RepoRoot, "TestAssets", "ProjectModelServer", "DthTestProjects", "src"));
         }
@@ -118,7 +121,7 @@ namespace Microsoft.DotNet.ProjectModel.Server.Tests
                 Console.WriteLine("Test is skipped on Linux");
                 return;
             }
-            
+
             var projectPath = Path.Combine(_testAssetsManager.AssetsRoot, testProjectName);
             Assert.NotNull(projectPath);
 
@@ -292,7 +295,7 @@ namespace Microsoft.DotNet.ProjectModel.Server.Tests
             var testAssetsPath = Path.Combine(RepoRoot, "TestAssets", "ProjectModelServer");
             var assetsManager = new TestAssetsManager(testAssetsPath);
             var testSource = assetsManager.CreateTestInstance("IncorrectProjectJson").TestRoot;
-            
+
             using (var server = new DthTestServer(_loggerFactory))
             using (var client = new DthTestClient(server))
             {
@@ -337,30 +340,73 @@ namespace Microsoft.DotNet.ProjectModel.Server.Tests
                         .AssertProperty<string>("Path", v => v.Contains("InvalidGlobalJson"));
             }
         }
-        
+
         [Fact]
         public void RecoverFromGlobalError()
         {
             var testProject = _testAssetsManager.CreateTestInstance("EmptyConsoleApp")
                                                 .WithLockFiles()
                                                 .TestRoot;
-                                                
+
             using (var server = new DthTestServer(_loggerFactory))
             using (var client = new DthTestClient(server))
             {
                 var projectFile = Path.Combine(testProject, Project.FileName);
                 var content = File.ReadAllText(projectFile);
                 File.WriteAllText(projectFile, content + "}");
-                
+
                 client.Initialize(testProject);
                 var messages = client.DrainAllMessages();
                 messages.ContainsMessage(MessageTypes.Error);
-                
+
                 File.WriteAllText(projectFile, content);
                 client.SendPayLoad(testProject, MessageTypes.FilesChanged);
                 var clearError = client.DrainTillFirst(MessageTypes.Error);
                 clearError.Payload.AsJObject().AssertProperty("Message", null as string);
             }
         }
+
+        [Theory]
+        [InlineData(500, true)]
+        [InlineData(3000, false)]
+        public void WaitForLockFileReleased(int occupyFileFor, bool expectSuccess)
+        {
+            var testProject = _testAssetsManager.CreateTestInstance("EmptyConsoleApp")
+                                                .WithLockFiles()
+                                                .TestRoot;
+
+            using (var server = new DthTestServer(_loggerFactory))
+            using (var client = new DthTestClient(server))
+            {
+                var lockFilePath = Path.Combine(testProject, LockFile.FileName);
+                var lockFileContent = File.ReadAllText(lockFilePath);
+                var fs = new FileStream(lockFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
+
+                // Test the platform
+                // A sharing violation is expected in following code. Otherwise the FileSteam is not implemented correctly.
+                Assert.ThrowsAny<IOException>(() =>
+                {
+                    new FileStream(lockFilePath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                });
+
+                var task = Task.Run(() =>
+                {
+                    // WorkspaceContext will try to open the lock file for 3 times with 500 ms interval in between.
+                    Thread.Sleep(occupyFileFor);
+                    fs.Dispose();
+                });
+
+                client.Initialize(testProject);
+                var messages = client.DrainAllMessages();
+                if (expectSuccess)
+                {
+                    messages.AssertDoesNotContain(MessageTypes.Error);
+                }
+                else
+                {
+                    messages.ContainsMessage(MessageTypes.Error);
+                }
+            }
+        }
     }
 }