From b6e3224387e6d66fac982250d88783ae4289657d Mon Sep 17 00:00:00 2001 From: Ajay Bhargav Baaskaran Date: Wed, 4 May 2016 00:41:01 -0700 Subject: [PATCH] [Fixes #2829 #2861] Maintain folder structure in mappings --- .../Files/IncludeFilesResolver.cs | 78 +++++++++++----- .../Utilities/PathUtility.cs | 25 ++++-- .../commands/dotnet-pack/PackageGenerator.cs | 2 +- ...ThatIWantToCreateIncludeEntriesFromJson.cs | 90 +++++++++++++++++++ test/dotnet-pack.Tests/PackTests.cs | 2 +- 5 files changed, 165 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.DotNet.ProjectModel/Files/IncludeFilesResolver.cs b/src/Microsoft.DotNet.ProjectModel/Files/IncludeFilesResolver.cs index 7d7e6e799..15ff88670 100644 --- a/src/Microsoft.DotNet.ProjectModel/Files/IncludeFilesResolver.cs +++ b/src/Microsoft.DotNet.ProjectModel/Files/IncludeFilesResolver.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using Microsoft.DotNet.ProjectModel.Utilities; using Microsoft.DotNet.ProjectModel.FileSystemGlobbing; +using Microsoft.DotNet.ProjectModel.FileSystemGlobbing.Abstractions; namespace Microsoft.DotNet.ProjectModel.Files { @@ -39,7 +40,7 @@ namespace Microsoft.DotNet.ProjectModel.Files sourceBasePath, DiagnosticMessageSeverity.Error)); } - else if (targetBasePath.Split('/').Any(s => s.Equals(".") || s.Equals(".."))) + else if (targetBasePath.Split(Path.DirectorySeparatorChar).Any(s => s.Equals(".") || s.Equals(".."))) { diagnostics?.Add(new DiagnosticMessage( ErrorCodes.NU1004, @@ -56,12 +57,11 @@ namespace Microsoft.DotNet.ProjectModel.Files context.IncludePatterns, context.ExcludePatterns, context.IncludeFiles, - context.ExcludeFiles, context.BuiltInsInclude, - context.BuiltInsExclude); + context.BuiltInsExclude).ToList(); var isFile = targetBasePath[targetBasePath.Length - 1] != Path.DirectorySeparatorChar; - if (isFile && files.Count() > 1) + if (isFile && files.Count > 1) { // It's a file. But the glob matched multiple things diagnostics?.Add(new DiagnosticMessage( @@ -72,30 +72,70 @@ namespace Microsoft.DotNet.ProjectModel.Files sourceBasePath, DiagnosticMessageSeverity.Error)); } - else if (isFile && files.Any()) + else if (isFile && files.Count > 0) { - includeEntries.Add(new IncludeEntry(targetBasePath, files.First())); + var filePath = Path.GetFullPath( + Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator(files[0].Path))); + + includeEntries.Add(new IncludeEntry(targetBasePath, filePath)); } - else + else if (!isFile) { targetBasePath = targetBasePath.Substring(0, targetBasePath.Length - 1); foreach (var file in files) { - string targetPath = null; + var fullPath = Path.GetFullPath( + Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator(file.Path))); + string targetPath; if (flatten) { - targetPath = Path.Combine(targetBasePath, Path.GetFileName(file)); + targetPath = Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator(file.Stem)); } else { - targetPath = Path.Combine(targetBasePath, PathUtility.GetRelativePath(sourceBasePath, file)); + targetPath = Path.Combine( + targetBasePath, + PathUtility.GetRelativePathIgnoringDirectoryTraversals(sourceBasePath, fullPath)); } - includeEntries.Add(new IncludeEntry(targetPath, file)); + includeEntries.Add(new IncludeEntry(targetPath, fullPath)); } } + + if (context.IncludeFiles != null) + { + foreach (var literalRelativePath in context.IncludeFiles) + { + var fullPath = Path.GetFullPath(Path.Combine(sourceBasePath, literalRelativePath)); + string targetPath; + + if (isFile) + { + targetPath = targetBasePath; + } + else if (flatten) + { + targetPath = Path.Combine(targetBasePath, Path.GetFileName(fullPath)); + } + else + { + targetPath = Path.Combine(targetBasePath, PathUtility.GetRelativePath(sourceBasePath, fullPath)); + } + + includeEntries.Add(new IncludeEntry(targetPath, fullPath)); + } + } + + if (context.ExcludeFiles != null) + { + var literalExcludedFiles = new HashSet( + context.ExcludeFiles.Select(file => Path.GetFullPath(Path.Combine(sourceBasePath, file))), + StringComparer.Ordinal); + + includeEntries.RemoveWhere(entry => literalExcludedFiles.Contains(entry.SourcePath)); + } } if (context.Mappings != null) @@ -105,7 +145,7 @@ namespace Microsoft.DotNet.ProjectModel.Files { var targetPath = Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator(map.Key)); - foreach (var file in GetIncludeFiles(map.Value, targetPath, diagnostics, flatten)) + foreach (var file in GetIncludeFiles(map.Value, targetPath, diagnostics, flatten: true)) { file.IsCustomTarget = true; @@ -119,12 +159,11 @@ namespace Microsoft.DotNet.ProjectModel.Files return includeEntries; } - private static IEnumerable GetIncludeFilesCore( + private static IEnumerable GetIncludeFilesCore( string sourceBasePath, List includePatterns, List excludePatterns, List includeFiles, - List excludeFiles, List builtInsInclude, List builtInsExclude) { @@ -167,16 +206,7 @@ namespace Microsoft.DotNet.ProjectModel.Files matcher.AddExcludePatterns(excludePatterns); } - var files = matcher.GetResultsInFullPath(sourceBasePath); - files = files.Concat(literalIncludedFiles).Distinct(); - - if (files.Any() && excludeFiles != null) - { - var literalExcludedFiles = excludeFiles.Select(file => Path.GetFullPath(Path.Combine(sourceBasePath, file))); - files = files.Except(literalExcludedFiles); - } - - return files; + return matcher.Execute(new DirectoryInfoWrapper(new DirectoryInfo(sourceBasePath))).Files; } } } diff --git a/src/Microsoft.DotNet.ProjectModel/Utilities/PathUtility.cs b/src/Microsoft.DotNet.ProjectModel/Utilities/PathUtility.cs index d5ea5d44a..dfdc79214 100644 --- a/src/Microsoft.DotNet.ProjectModel/Utilities/PathUtility.cs +++ b/src/Microsoft.DotNet.ProjectModel/Utilities/PathUtility.cs @@ -61,17 +61,26 @@ namespace Microsoft.DotNet.ProjectModel.Utilities } /// - /// Returns path2 relative to path1, with Path.DirectorySeparatorChar as separator + /// Returns relative to , with Path.DirectorySeparatorChar as separator /// public static string GetRelativePath(string path1, string path2) { - return GetRelativePath(path1, path2, Path.DirectorySeparatorChar); + return GetRelativePath(path1, path2, Path.DirectorySeparatorChar, includeDirectoryTraversals: true); } /// - /// Returns path2 relative to path1, with given path separator + /// Returns relative to , with Path.DirectorySeparatorChar + /// as separator but ignoring directory traversals. /// - public static string GetRelativePath(string path1, string path2, char separator) + public static string GetRelativePathIgnoringDirectoryTraversals(string path1, string path2) + { + return GetRelativePath(path1, path2, Path.DirectorySeparatorChar, false); + } + + /// + /// Returns relative to , with given path separator + /// + public static string GetRelativePath(string path1, string path2, char separator, bool includeDirectoryTraversals) { if (string.IsNullOrEmpty(path1)) { @@ -133,10 +142,14 @@ namespace Microsoft.DotNet.ProjectModel.Utilities return path; } - for (var i = index; len1 > i; ++i) + if (includeDirectoryTraversals) { - path += ".." + separator; + for (var i = index; len1 > i; ++i) + { + path += ".." + separator; + } } + for (var i = index; len2 - 1 > i; ++i) { path += path2Segments[i] + separator; diff --git a/src/dotnet/commands/dotnet-pack/PackageGenerator.cs b/src/dotnet/commands/dotnet-pack/PackageGenerator.cs index f0fe7e804..4b4f891aa 100644 --- a/src/dotnet/commands/dotnet-pack/PackageGenerator.cs +++ b/src/dotnet/commands/dotnet-pack/PackageGenerator.cs @@ -123,7 +123,7 @@ namespace Microsoft.DotNet.Tools.Compiler if (Project.PackOptions.PackInclude != null) { - var files = IncludeFilesResolver.GetIncludeFiles(Project.PackOptions.PackInclude, "/", diagnostics: packDiagnostics, flatten: true); + var files = IncludeFilesResolver.GetIncludeFiles(Project.PackOptions.PackInclude, "/", diagnostics: packDiagnostics); PackageBuilder.Files.AddRange(GetPackageFiles(files, packDiagnostics)); } else if (Project.Files.PackInclude != null && Project.Files.PackInclude.Any()) diff --git a/test/Microsoft.DotNet.ProjectModel.Tests/GivenThatIWantToCreateIncludeEntriesFromJson.cs b/test/Microsoft.DotNet.ProjectModel.Tests/GivenThatIWantToCreateIncludeEntriesFromJson.cs index 5f6bd787c..b02e9a48a 100644 --- a/test/Microsoft.DotNet.ProjectModel.Tests/GivenThatIWantToCreateIncludeEntriesFromJson.cs +++ b/test/Microsoft.DotNet.ProjectModel.Tests/GivenThatIWantToCreateIncludeEntriesFromJson.cs @@ -173,6 +173,96 @@ namespace Microsoft.DotNet.ProjectModel.Tests entry.SourcePath.Contains(PathUtility.GetPathWithDirectorySeparator("files/pfile2ex.txt"))); } + [Fact] + public void It_maintains_folder_structure() + { + var json = JObject.Parse(@"{ +'packOptions': { + 'files': { + 'include': 'packfiles/**/*.txt', + 'excludeFiles': 'packfiles/morefiles/file2.txt', + 'mappings': { + 'somepath/': 'mappackfiles/**/*.txt' + } + } +}}"); + + CreateFile("packfiles/morefiles/file1.txt"); + CreateFile("packfiles/morefiles/file2.txt"); + CreateFile("packfiles/file3.txt"); + CreateFile("mappackfiles/morefiles/file4.txt"); + CreateFile("mappackfiles/morefiles/file5.txt"); + CreateFile("mappackfiles/file6.txt"); + + var project = GetProject(json); + var sourceBasePath = project.PackOptions.PackInclude.SourceBasePath; + var targetBasePath = PathUtility.GetPathWithDirectorySeparator("basepath/"); + + var packIncludeEntries = GetIncludeFiles(project.PackOptions.PackInclude, targetBasePath); + + packIncludeEntries.Should().HaveCount(5); + + packIncludeEntries.Should().Contain(entry => + entry.SourcePath == Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("packfiles/morefiles/file1.txt")) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("packfiles/morefiles/file1.txt"))); + packIncludeEntries.Should().Contain(entry => + entry.SourcePath == Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("packfiles/file3.txt")) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("packfiles/file3.txt"))); + packIncludeEntries.Should().Contain(entry => + entry.SourcePath == Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("mappackfiles/morefiles/file4.txt")) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/morefiles/file4.txt"))); + packIncludeEntries.Should().Contain(entry => + entry.SourcePath == Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("mappackfiles/morefiles/file5.txt")) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/morefiles/file5.txt"))); + packIncludeEntries.Should().Contain(entry => + entry.SourcePath == Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("mappackfiles/file6.txt")) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/file6.txt"))); + } + + [Fact] + public void It_handles_paths_with_directory_traversals_properly() + { + var json = JObject.Parse(@"{ +'publishOptions': { + 'include': '../pubfiles/**/*.txt', + 'excludeFiles': '../pubfiles/morefiles/file2.txt', + 'mappings': { + 'somepath/': '../mappubfiles/**/*.txt' + } +}}"); + + CreateFile("../pubfiles/morefiles/file1.txt"); + CreateFile("../pubfiles/morefiles/file2.txt"); + CreateFile("../pubfiles/file3.txt"); + CreateFile("../mappubfiles/morefiles/file4.txt"); + CreateFile("../mappubfiles/morefiles/file5.txt"); + CreateFile("../mappubfiles/file6.txt"); + + var project = GetProject(json); + var sourceBasePath = project.PublishOptions.SourceBasePath; + var targetBasePath = PathUtility.GetPathWithDirectorySeparator("basepath/"); + + var publishEntries = GetIncludeFiles(project.PublishOptions, targetBasePath); + + publishEntries.Should().HaveCount(5); + + publishEntries.Should().Contain(entry => + entry.SourcePath == Path.GetFullPath(Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("../pubfiles/morefiles/file1.txt"))) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("pubfiles/morefiles/file1.txt"))); + publishEntries.Should().Contain(entry => + entry.SourcePath == Path.GetFullPath(Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("../pubfiles/file3.txt"))) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("pubfiles/file3.txt"))); + publishEntries.Should().Contain(entry => + entry.SourcePath == Path.GetFullPath(Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("../mappubfiles/morefiles/file4.txt"))) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/morefiles/file4.txt"))); + publishEntries.Should().Contain(entry => + entry.SourcePath == Path.GetFullPath(Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("../mappubfiles/morefiles/file5.txt"))) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/morefiles/file5.txt"))); + publishEntries.Should().Contain(entry => + entry.SourcePath == Path.GetFullPath(Path.Combine(sourceBasePath, PathUtility.GetPathWithDirectorySeparator("../mappubfiles/file6.txt"))) && + entry.TargetPath == Path.Combine(targetBasePath, PathUtility.GetPathWithDirectorySeparator("somepath/file6.txt"))); + } + private Project GetProject(JObject json, ProjectReaderSettings settings = null) { using (var stream = new MemoryStream()) diff --git a/test/dotnet-pack.Tests/PackTests.cs b/test/dotnet-pack.Tests/PackTests.cs index d5e3aadbd..7613ed159 100644 --- a/test/dotnet-pack.Tests/PackTests.cs +++ b/test/dotnet-pack.Tests/PackTests.cs @@ -109,7 +109,7 @@ namespace Microsoft.DotNet.Tools.Compiler.Tests File.Exists(outputPackage).Should().BeTrue(outputPackage); var zip = ZipFile.Open(outputPackage, ZipArchiveMode.Read); - zip.Entries.Should().Contain(e => e.FullName == "pack1.txt"); + zip.Entries.Should().Contain(e => e.FullName == "packfiles/pack1.txt"); zip.Entries.Should().Contain(e => e.FullName == "newpath/pack2.txt"); }