diff --git a/build_projects/dotnet-cli-build/PublishTargets.cs b/build_projects/dotnet-cli-build/PublishTargets.cs
index 74cbf6619..a3e908f0a 100644
--- a/build_projects/dotnet-cli-build/PublishTargets.cs
+++ b/build_projects/dotnet-cli-build/PublishTargets.cs
@@ -73,9 +73,12 @@ namespace Microsoft.DotNet.Cli.Build
                 }
                 else
                 {
-                    // This is an old drop of latest so remove all old files to ensure a clean state
+                    Regex versionFileRegex = new Regex(@"(?<version>\d\.\d\.\d)-(?<release>.*)?");
+
+                    // Delete old version files
                     AzurePublisherTool.ListBlobs($"{targetContainer}")
                         .Select(s => s.Replace("/dotnet/", ""))
+                        .Where(s => versionFileRegex.IsMatch(s))
                         .ToList()
                         .ForEach(f => AzurePublisherTool.TryDeleteBlob(f));