Binary tooling (#18726)
This commit is contained in:
Ella Hathaway
GitHub
parent
15f666742c
commit
b76440930b
12 changed files with 753 additions and 41 deletions
|
|
@ -386,6 +386,7 @@ jobs:
|
|||
find artifacts/ -type f -name "*.diff" -exec rsync -R {} -t ${targetFolder} \;
|
||||
if [[ "${{ parameters.buildSourceOnly }}" == "True" ]]; then
|
||||
find artifacts/prebuilt-report/ -exec rsync -R {} -t ${targetFolder} \;
|
||||
find artifacts/log/binary-report/ -exec rsync -R {} -t ${targetFolder} \;
|
||||
fi
|
||||
find src/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
|
||||
find src/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
|
||||
|
|
|
|||
|
|
@ -2,8 +2,9 @@
|
|||
|
||||
### Usage: $0
|
||||
###
|
||||
### Prepares the environment for a source build by downloading Private.SourceBuilt.Artifacts.*.tar.gz and
|
||||
### installing the version of dotnet referenced in global.json
|
||||
### Prepares the environment for a source build by downloading Private.SourceBuilt.Artifacts.*.tar.gz,
|
||||
### installing the version of dotnet referenced in global.json,
|
||||
### and detecting binaries and removing any non-SB allowed binaries.
|
||||
###
|
||||
### Options:
|
||||
### --no-artifacts Exclude the download of the previously source-built artifacts archive
|
||||
|
|
@ -15,6 +16,21 @@
|
|||
### --runtime-source-feed URL of a remote server or a local directory, from which SDKs and
|
||||
### runtimes can be downloaded
|
||||
### --runtime-source-feed-key Key for accessing the above server, if necessary
|
||||
###
|
||||
### Binary-Tooling options:
|
||||
### --no-binary-tooling Don't run the binary tooling
|
||||
### --allowed-binaries Path to the file containing the list of known binaries that are allowed
|
||||
### in the VMR and can be kept for source-building.
|
||||
### Default is src/installer/src/VirtualMonoRepo/allowed-binaries.txt
|
||||
### --disallowed-sb-binaries Path to the file containing the list of known binaries that are allowed
|
||||
### in the VMR but cannot be kept for source-building.
|
||||
### Default is null.
|
||||
### --with-sdk Use the SDK in the specified directory
|
||||
### Default is the .NET SDK
|
||||
### --with-packages URL or specified directory to use as the source feed for packages
|
||||
### Default is the previously source-built artifacts archive
|
||||
### --no-validate Do not run validation. Only remove the binaries.
|
||||
### --no-clean Do not remove the binaries. Only run the validation.
|
||||
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
|
@ -26,8 +42,16 @@ function print_help () {
|
|||
sed -n '/^### /,/^$/p' "$source" | cut -b 5-
|
||||
}
|
||||
|
||||
# SB prep default arguments
|
||||
defaultArtifactsRid='centos.8-x64'
|
||||
|
||||
# Binary Tooling default arguments
|
||||
defaultAllowedBinaries="$REPO_ROOT/src/installer/src/VirtualMonoRepo/allowed-binaries.txt"
|
||||
defaultDotnetSdk="$REPO_ROOT/.dotnet"
|
||||
defaultPackagesDir="$REPO_ROOT/prereqs/packages"
|
||||
defaultMode="All"
|
||||
|
||||
# SB prep arguments
|
||||
buildBootstrap=true
|
||||
downloadArtifacts=true
|
||||
downloadPrebuilts=true
|
||||
|
|
@ -35,6 +59,15 @@ installDotnet=true
|
|||
artifactsRid=$defaultArtifactsRid
|
||||
runtime_source_feed='' # IBM requested these to support s390x scenarios
|
||||
runtime_source_feed_key='' # IBM requested these to support s390x scenarios
|
||||
|
||||
# Binary Tooling arguments
|
||||
runBinaryTool=true
|
||||
allowedBinaries=$defaultAllowedBinaries
|
||||
disallowedSbBinaries=''
|
||||
dotnetSdk=$defaultDotnetSdk
|
||||
packagesSourceFeed=$defaultPackagesDir
|
||||
mode=$defaultMode
|
||||
|
||||
positional_args=()
|
||||
while :; do
|
||||
if [ $# -le 0 ]; then
|
||||
|
|
@ -69,6 +102,47 @@ while :; do
|
|||
runtime_source_feed_key=$2
|
||||
shift
|
||||
;;
|
||||
--no-binary-tooling)
|
||||
runBinaryTool=false
|
||||
;;
|
||||
--allowed-binaries)
|
||||
allowedBinaries=$2
|
||||
if [ ! -f "$allowedBinaries" ]; then
|
||||
echo "Allowed binaries file '$allowedBinaries' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--disallowed-sb-binaries)
|
||||
disallowedSbBinaries=$2
|
||||
if [ ! -f "$disallowedSbBinaries" ]; then
|
||||
echo "Disallowed source build binaries file '$disallowedSbBinaries' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--with-sdk)
|
||||
dotnetSdk=$2
|
||||
if [ ! -d "$dotnetSdk" ]; then
|
||||
echo "Custom SDK directory '$dotnetSdk' does not exist"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -x "$dotnetSdk/dotnet" ]; then
|
||||
echo "Custom SDK '$dotnetSdk/dotnet' does not exist or is not executable"
|
||||
exit 1
|
||||
fi
|
||||
shift
|
||||
;;
|
||||
--with-packages)
|
||||
packagesSourceFeed=$2
|
||||
shift
|
||||
;;
|
||||
--no-clean)
|
||||
mode="Validate"
|
||||
;;
|
||||
--no-validate)
|
||||
mode="Clean"
|
||||
;;
|
||||
*)
|
||||
positional_args+=("$1")
|
||||
;;
|
||||
|
|
@ -112,6 +186,56 @@ if [ "$installDotnet" == true ] && [ -d "$REPO_ROOT/.dotnet" ]; then
|
|||
installDotnet=false;
|
||||
fi
|
||||
|
||||
function ParseBinaryArgs {
|
||||
# Attempting to run the binary tooling without an SDK will fail. So either the --with-sdk flag must be passed
|
||||
# or a pre-existing .dotnet SDK directory must exist.
|
||||
if [ "$dotnetSdk" == "$defaultDotnetSdk" ] && [ ! -d "$dotnetSdk" ]; then
|
||||
echo " ERROR: A pre-existing .dotnet SDK directory is needed if --with-sdk is not provided. \
|
||||
Please either supply an SDK using --with-sdk or execute ./eng/prep-source-build.sh before proceeding. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
## Attemping to run the binary tooling without a packages directory or source-feed will fail. So either the
|
||||
## --with-packages flag must be passed with a valid directory or a pre-existing packages directory must exist.
|
||||
if [ "$packagesSourceFeed" == "$defaultPackagesDir" ] && [ ! -d "$packagesSourceFeed" ]; then
|
||||
echo " ERROR: A pre-existing packages directory is needed if --with-packages is not provided. \
|
||||
Please either supply a packages directory using --with-packages or \
|
||||
execute ./eng/prep-source-build.sh with download artifacts enabled before proceeding. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Attempting to run the binary tooling with a custom packages feed that does not
|
||||
# have PackageVersions.props in the packages directory or source-feed will fail.
|
||||
if [ "$packagesSourceFeed" != "$defaultPackagesDir" ] && [ ! -f "$packagesSourceFeed/PackageVersions.props" ]; then
|
||||
echo " ERROR: PackageVersions.props is needed in the packages directory or source-feed. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Set up the packages source feed if we're using the default artifacts
|
||||
previouslyBuiltPackagesDir="$defaultPackagesDir/previously-source-built"
|
||||
packageArtifacts="$defaultPackagesDir/archive/Private.SourceBuilt.Artifacts.*.tar.gz"
|
||||
if [ "$packagesSourceFeed" == "$defaultPackagesDir" ]; then
|
||||
if [ -d "$previouslyBuiltPackagesDir" ]; then
|
||||
echo " Previously source built packages directory exists..."
|
||||
echo " Using $previouslyBuiltPackagesDir as the source-feed for the binary tooling..."
|
||||
packagesSourceFeed="$previouslyBuiltPackagesDir"
|
||||
elif [ -f ${packageArtifacts} ]; then
|
||||
echo " Unpacking Private.SourceBuilt.Artifacts.*.tar.gz to $previouslyBuiltPackagesDir..."
|
||||
mkdir -p "$previouslyBuiltPackagesDir"
|
||||
tar -xzf ${packageArtifacts} -C "$previouslyBuiltPackagesDir"
|
||||
tar -xzf ${packageArtifacts} -C "$previouslyBuiltPackagesDir" PackageVersions.props
|
||||
|
||||
echo " Using $previouslyBuiltPackagesDir as the source-feed for the binary tooling..."
|
||||
packagesSourceFeed="$previouslyBuiltPackagesDir"
|
||||
else
|
||||
echo " ERROR: A pre-existing package archive is needed if --with-packages is not provided. \
|
||||
Please either supply a source-feed using --with-packages or execute ./eng/prep-source-build.sh \
|
||||
with download artifacts enabled before proceeding. Exiting..."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function DownloadArchive {
|
||||
archiveType="$1"
|
||||
isRequired="$2"
|
||||
|
|
@ -171,6 +295,18 @@ function BootstrapArtifacts {
|
|||
rm -rf "$workingDir"
|
||||
}
|
||||
|
||||
function RunBinaryTool {
|
||||
BinaryTool="$REPO_ROOT/eng/tools/BinaryToolKit"
|
||||
TargetDir="$REPO_ROOT"
|
||||
OutputDir="$REPO_ROOT/artifacts/log/binary-report"
|
||||
|
||||
# Set the environment variable for the packages source feed
|
||||
export ARTIFACTS_PATH="$packagesSourceFeed"
|
||||
|
||||
# Run the BinaryDetection tool
|
||||
"$dotnetSdk/dotnet" run --project "$BinaryTool" -c Release -p PackagesPropsDirectory="$packagesSourceFeed" "$TargetDir" "$OutputDir" -ab "$allowedBinaries" -db "$disallowedSbBinaries" -m $mode -l Debug
|
||||
}
|
||||
|
||||
# Check for the version of dotnet to install
|
||||
if [ "$installDotnet" == true ]; then
|
||||
echo " Installing dotnet..."
|
||||
|
|
@ -189,3 +325,8 @@ fi
|
|||
if [ "$downloadPrebuilts" == true ]; then
|
||||
DownloadArchive Prebuilts false $artifactsRid
|
||||
fi
|
||||
|
||||
if [ "$runBinaryTool" == true ]; then
|
||||
ParseBinaryArgs
|
||||
RunBinaryTool
|
||||
fi
|
||||
100
src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryTool.cs
Normal file
100
src/SourceBuild/content/eng/tools/BinaryToolKit/BinaryTool.cs
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public class BinaryTool
|
||||
{
|
||||
public async Task ExecuteAsync(
|
||||
string targetDirectory,
|
||||
string outputReportDirectory,
|
||||
string? allowedBinariesFile,
|
||||
string? disallowedSbBinariesFile,
|
||||
Modes mode)
|
||||
{
|
||||
DateTime startTime = DateTime.Now;
|
||||
|
||||
Log.LogInformation($"Starting binary tool at {startTime} in {mode} mode");
|
||||
|
||||
// Parse args
|
||||
targetDirectory = GetAndValidateFullPath(
|
||||
"TargetDirectory",
|
||||
targetDirectory,
|
||||
isDirectory: true,
|
||||
createIfNotExist: false,
|
||||
isRequired: true)!;
|
||||
outputReportDirectory = GetAndValidateFullPath(
|
||||
"OutputReportDirectory",
|
||||
outputReportDirectory,
|
||||
isDirectory: true,
|
||||
createIfNotExist: true,
|
||||
isRequired: true)!;
|
||||
allowedBinariesFile = GetAndValidateFullPath(
|
||||
"AllowedBinariesFile",
|
||||
allowedBinariesFile,
|
||||
isDirectory: false,
|
||||
createIfNotExist: false,
|
||||
isRequired: false);
|
||||
disallowedSbBinariesFile = GetAndValidateFullPath(
|
||||
"DisallowedSbBinariesFile",
|
||||
disallowedSbBinariesFile,
|
||||
isDirectory: false,
|
||||
createIfNotExist: false,
|
||||
isRequired: false);
|
||||
|
||||
// Run the tooling
|
||||
var detectedBinaries = await DetectBinaries.ExecuteAsync(targetDirectory);
|
||||
|
||||
var comparedBinaries = CompareBinariesAgainstBaselines
|
||||
.Execute(
|
||||
detectedBinaries,
|
||||
allowedBinariesFile,
|
||||
disallowedSbBinariesFile,
|
||||
outputReportDirectory,
|
||||
targetDirectory,
|
||||
mode);
|
||||
|
||||
if (mode.HasFlag(Modes.Clean))
|
||||
{
|
||||
RemoveBinaries.Execute(comparedBinaries, targetDirectory);
|
||||
}
|
||||
|
||||
Log.LogInformation("Finished all binary tasks. Took " + (DateTime.Now - startTime).TotalSeconds + " seconds.");
|
||||
}
|
||||
|
||||
private string? GetAndValidateFullPath(
|
||||
string parameterName,
|
||||
string? path,
|
||||
bool isDirectory,
|
||||
bool createIfNotExist,
|
||||
bool isRequired)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
if (isRequired)
|
||||
{
|
||||
Log.LogError($"Required path for '{parameterName}' is empty or contains whitespace.");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
string fullPath = Path.GetFullPath(path);
|
||||
bool exists = isDirectory ? Directory.Exists(fullPath) : File.Exists(fullPath);
|
||||
|
||||
if (!exists)
|
||||
{
|
||||
if (createIfNotExist && isDirectory)
|
||||
{
|
||||
Log.LogInformation($"Creating directory '{fullPath}' for '{parameterName}'.");
|
||||
Directory.CreateDirectory(fullPath);
|
||||
}
|
||||
else
|
||||
{
|
||||
Log.LogError($"{(isDirectory ? "Directory" : "File")} '{fullPath}' for '{parameterName}' does not exist.");
|
||||
Environment.Exit(1);
|
||||
}
|
||||
}
|
||||
return fullPath;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<Project Sdk="Microsoft.NET.Sdk" InitialTargets="CheckPackagesPropsDirectory">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>$(NetCurrent)</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<PackagesPropsDirectory></PackagesPropsDirectory>
|
||||
<OutputType>Exe</OutputType>
|
||||
</PropertyGroup>
|
||||
|
||||
<Target Name="CheckPackagesPropsDirectory">
|
||||
<Error Condition="'$(PackagesPropsDirectory)' == ''" Text="PackagesPropsDirectory is not set. Please pass as an MSBuild Property" />
|
||||
<Error Condition="!Exists('$(PackagesPropsDirectory)/PackageVersions.props')" Text="PackageVersions.props does not exist in $(PackagesPropsDirectory)." />
|
||||
</Target>
|
||||
|
||||
<!-- Need to condition this import because msbuild will complain about the project not being valid otherwise. -->
|
||||
<!-- With the condition, the CheckPackagesPropsDirectory will run as expected and show the respective errors. -->
|
||||
<Import Project="$(PackagesPropsDirectory)/PackageVersions.props" Condition="'$(PackagesPropsDirectory)' != ''" />
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" />
|
||||
<PackageReference Include="System.CommandLine" Version="$(SystemCommandLineVersion)" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
using Microsoft.Extensions.FileSystemGlobbing;
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public static class CompareBinariesAgainstBaselines
|
||||
{
|
||||
public static List<string> Execute(
|
||||
IEnumerable<string> detectedBinaries,
|
||||
string? allowedBinariesFile,
|
||||
string? disallowedSbBinariesFile,
|
||||
string outputReportDirectory,
|
||||
string targetDirectory,
|
||||
Modes mode)
|
||||
{
|
||||
Log.LogInformation("Comparing detected binaries to baseline(s).");
|
||||
|
||||
var binariesToRemove = GetUnmatchedBinaries(
|
||||
detectedBinaries,
|
||||
allowedBinariesFile,
|
||||
outputReportDirectory,
|
||||
targetDirectory,
|
||||
mode).ToList();
|
||||
|
||||
if (mode.HasFlag(Modes.Validate))
|
||||
{
|
||||
var nonSbBinariesToRemove = GetUnmatchedBinaries(
|
||||
detectedBinaries,
|
||||
disallowedSbBinariesFile,
|
||||
outputReportDirectory,
|
||||
targetDirectory,
|
||||
mode).ToList();
|
||||
|
||||
var newBinaries = binariesToRemove.Intersect(nonSbBinariesToRemove);
|
||||
|
||||
if (newBinaries.Any())
|
||||
{
|
||||
string newBinariesFile = Path.Combine(outputReportDirectory, "NewBinaries.txt");
|
||||
|
||||
File.WriteAllLines(newBinariesFile, newBinaries);
|
||||
|
||||
Log.LogWarning($" {newBinaries.Count()} new binaries detected. Check '{newBinariesFile}' for details.");
|
||||
}
|
||||
}
|
||||
|
||||
Log.LogInformation("Finished comparing binaries.");
|
||||
|
||||
return binariesToRemove;
|
||||
}
|
||||
|
||||
private static IEnumerable<string> GetUnmatchedBinaries(
|
||||
IEnumerable<string> searchFiles,
|
||||
string? baselineFile,
|
||||
string outputReportDirectory,
|
||||
string targetDirectory,
|
||||
Modes mode)
|
||||
{
|
||||
var patterns = ParseBaselineFile(baselineFile);
|
||||
|
||||
if (mode.HasFlag(Modes.Validate))
|
||||
{
|
||||
// If validating in any mode (Mode == Validate or Mode == All),
|
||||
// we need to detect both unused patterns and unmatched files.
|
||||
// We simultaneously detect unused patterns and unmatched files for efficiency.
|
||||
|
||||
HashSet<string> unusedPatterns = new HashSet<string>(patterns);
|
||||
HashSet<string> unmatchedFiles = new HashSet<string>(searchFiles);
|
||||
|
||||
foreach (string pattern in patterns)
|
||||
{
|
||||
Matcher matcher = new Matcher(StringComparison.Ordinal);
|
||||
matcher.AddInclude(pattern);
|
||||
|
||||
var matches = matcher.Match(targetDirectory, searchFiles);
|
||||
if (matches.HasMatches)
|
||||
{
|
||||
unusedPatterns.Remove(pattern);
|
||||
unmatchedFiles.ExceptWith(matches.Files.Select(file => file.Path));
|
||||
}
|
||||
}
|
||||
|
||||
UpdateBaselineFile(baselineFile, outputReportDirectory, unusedPatterns);
|
||||
|
||||
return unmatchedFiles;
|
||||
}
|
||||
else if (mode == Modes.Clean)
|
||||
{
|
||||
// If only cleaning and not validating (Mode == Clean),
|
||||
// we don't need to update the baseline files with unused patterns
|
||||
// so we can just detect unmatched files.
|
||||
|
||||
Matcher matcher = new Matcher(StringComparison.Ordinal);
|
||||
matcher.AddInclude("**/*");
|
||||
matcher.AddExcludePatterns(patterns);
|
||||
|
||||
return matcher.Match(targetDirectory, searchFiles).Files.Select(file => file.Path);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Unhandled mode
|
||||
throw new ArgumentException($"Unhandled mode: {mode}");
|
||||
}
|
||||
}
|
||||
|
||||
private static IEnumerable<string> ParseBaselineFile(string? file) {
|
||||
if (!File.Exists(file))
|
||||
{
|
||||
return Enumerable.Empty<string>();
|
||||
}
|
||||
|
||||
// Read the baseline file and parse the patterns, ignoring comments and empty lines
|
||||
return File.ReadLines(file)
|
||||
.Select(line => line.Trim())
|
||||
.Where(line => !string.IsNullOrWhiteSpace(line) && !line.StartsWith("#"))
|
||||
.Select(line => line.Split('#')[0].Trim());
|
||||
}
|
||||
|
||||
private static void UpdateBaselineFile(string? file, string outputReportDirectory, HashSet<string> unusedPatterns)
|
||||
{
|
||||
if(File.Exists(file))
|
||||
{
|
||||
var lines = File.ReadAllLines(file);
|
||||
var newLines = lines.Where(line => !unusedPatterns.Contains(line)).ToList();
|
||||
|
||||
string updatedFile = Path.Combine(outputReportDirectory, "Updated" + Path.GetFileName(file));
|
||||
|
||||
File.WriteAllLines(updatedFile, newLines);
|
||||
|
||||
Log.LogInformation($" Updated baseline file '{file}' written to '{updatedFile}'");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.FileSystemGlobbing;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public static class DetectBinaries
|
||||
{
|
||||
private const string Utf16Marker = "UTF-16";
|
||||
private const int ChunkSize = 4096;
|
||||
private static readonly Regex GitCleanRegex = new Regex(@"Would (remove|skip)( repository)? (.*)");
|
||||
|
||||
public static async Task<List<string>> ExecuteAsync(string targetDirectory)
|
||||
{
|
||||
Log.LogInformation($"Detecting binaries in '{targetDirectory}'...");
|
||||
|
||||
var matcher = new Matcher(StringComparison.Ordinal);
|
||||
matcher.AddInclude("**/*");
|
||||
matcher.AddExcludePatterns(await GetIgnoredPatternsAsync(targetDirectory));
|
||||
|
||||
IEnumerable<string> matchingFiles = matcher.GetResultsInFullPath(targetDirectory);
|
||||
|
||||
var tasks = matchingFiles
|
||||
.Select(async file =>
|
||||
{
|
||||
return await IsBinaryAsync(file) ? file.Substring(targetDirectory.Length + 1) : null;
|
||||
});
|
||||
|
||||
var binaryFiles = (await Task.WhenAll(tasks)).OfType<string>().ToList();
|
||||
|
||||
Log.LogInformation($"Finished binary detection.");
|
||||
|
||||
return binaryFiles;
|
||||
}
|
||||
|
||||
private static async Task<List<string>> GetIgnoredPatternsAsync(string targetDirectory)
|
||||
{
|
||||
string gitDirectory = Path.Combine(targetDirectory, ".git");
|
||||
bool isGitRepo = Directory.Exists(gitDirectory);
|
||||
|
||||
try
|
||||
{
|
||||
if (!isGitRepo)
|
||||
{
|
||||
// Configure a fake git repo to use so that we can run git clean -ndx
|
||||
await ExecuteProcessAsync("git", $"-C {targetDirectory} init -q");
|
||||
}
|
||||
|
||||
await ExecuteProcessAsync("git", $"-C {targetDirectory} config --global safe.directory {targetDirectory}");
|
||||
|
||||
string output = await ExecuteProcessAsync("git", $"-C {targetDirectory} clean -ndx");
|
||||
|
||||
List<string> ignoredPaths = output.Split(Environment.NewLine)
|
||||
.Select(line => GitCleanRegex.Match(line))
|
||||
.Where(match => match.Success)
|
||||
.Select(match => match.Groups[3].Value)
|
||||
.ToList();
|
||||
|
||||
if (isGitRepo)
|
||||
{
|
||||
ignoredPaths.Add(".git");
|
||||
}
|
||||
|
||||
return ignoredPaths;
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Ensure .git directory is deleted if it wasn't originally a git repo
|
||||
if (!isGitRepo && Directory.Exists(gitDirectory))
|
||||
{
|
||||
Directory.Delete(gitDirectory, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<bool> IsBinaryAsync(string filePath)
|
||||
{
|
||||
// Using the GNU diff heuristic to determine if a file is binary or not.
|
||||
// For more details, refer to the GNU diff manual:
|
||||
// https://www.gnu.org/software/diffutils/manual/html_node/Binary.html
|
||||
|
||||
using (FileStream fs = new FileStream(filePath, FileMode.Open, FileAccess.Read))
|
||||
using (BinaryReader br = new BinaryReader(fs))
|
||||
{
|
||||
byte[] buffer = new byte[ChunkSize];
|
||||
int bytesRead = br.Read(buffer, 0, ChunkSize);
|
||||
for (int i = 0; i < bytesRead; i++)
|
||||
{
|
||||
if (buffer[i] == 0)
|
||||
{
|
||||
// Need to check that the file is not UTF-16 encoded
|
||||
// because heuristic can return false positives
|
||||
return await IsNotUTF16Async(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static async Task<bool> IsNotUTF16Async(string file)
|
||||
{
|
||||
if (Environment.OSVersion.Platform == PlatformID.Unix)
|
||||
{
|
||||
string output = await ExecuteProcessAsync("file", $"\"{file}\"");
|
||||
output = output.Split(":")[1].Trim();
|
||||
|
||||
if (output.Contains(Utf16Marker))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task<string> ExecuteProcessAsync(string executable, string arguments)
|
||||
{
|
||||
ProcessStartInfo psi = new ()
|
||||
{
|
||||
FileName = executable,
|
||||
Arguments = arguments,
|
||||
CreateNoWindow = true,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true
|
||||
};
|
||||
|
||||
var proc = Process.Start(psi)!;
|
||||
|
||||
string output = await proc.StandardOutput.ReadToEndAsync();
|
||||
string error = await proc.StandardError.ReadToEndAsync();
|
||||
|
||||
await proc.WaitForExitAsync();
|
||||
|
||||
if (!string.IsNullOrEmpty(error))
|
||||
{
|
||||
Log.LogError(error);
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
47
src/SourceBuild/content/eng/tools/BinaryToolKit/Log.cs
Normal file
47
src/SourceBuild/content/eng/tools/BinaryToolKit/Log.cs
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public static class Log
|
||||
{
|
||||
public static LogLevel Level = LogLevel.Information;
|
||||
|
||||
private static readonly Lazy<ILogger> _logger = new Lazy<ILogger>(ConfigureLogger);
|
||||
|
||||
public static void LogDebug(string message)
|
||||
{
|
||||
_logger.Value.LogDebug(message);
|
||||
}
|
||||
|
||||
public static void LogInformation(string message)
|
||||
{
|
||||
_logger.Value.LogInformation(message);
|
||||
}
|
||||
|
||||
public static void LogWarning(string message)
|
||||
{
|
||||
_logger.Value.LogWarning(message);
|
||||
}
|
||||
|
||||
public static void LogError(string message)
|
||||
{
|
||||
_logger.Value.LogError(message);
|
||||
}
|
||||
|
||||
private static ILogger ConfigureLogger()
|
||||
{
|
||||
using ILoggerFactory loggerFactory =
|
||||
LoggerFactory.Create(builder =>
|
||||
builder.AddSimpleConsole(options =>
|
||||
{
|
||||
options.SingleLine = true;
|
||||
options.TimestampFormat = "HH:mm:ss ";
|
||||
options.UseUtcTimestamp = true;
|
||||
})
|
||||
.SetMinimumLevel(Level));
|
||||
return loggerFactory.CreateLogger("BinaryTool");
|
||||
}
|
||||
}
|
||||
12
src/SourceBuild/content/eng/tools/BinaryToolKit/Modes.cs
Normal file
12
src/SourceBuild/content/eng/tools/BinaryToolKit/Modes.cs
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
[Flags]
|
||||
public enum Modes
|
||||
{
|
||||
Validate = 1,
|
||||
Clean = 2,
|
||||
All = Validate | Clean
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<configuration>
|
||||
<packageSources>
|
||||
<clear />
|
||||
<!-- %ARTIFACTS_PATH% must be set via an environment variable (export ARTIFACTS_PATH=<path-to-sources>). -->
|
||||
<add key="artifactsPath" value="%ARTIFACTS_PATH%" />
|
||||
</packageSources>
|
||||
</configuration>
|
||||
76
src/SourceBuild/content/eng/tools/BinaryToolKit/Program.cs
Normal file
76
src/SourceBuild/content/eng/tools/BinaryToolKit/Program.cs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
using System;
|
||||
using System.CommandLine;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public class Program
|
||||
{
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
CliArgument<string> TargetDirectory = new("target-directory")
|
||||
{
|
||||
Description = "The directory to run the binary tooling on."
|
||||
};
|
||||
|
||||
CliArgument<string> OutputReportDirectory = new("output-report-directory")
|
||||
{
|
||||
Description = "The directory to output the report to."
|
||||
};
|
||||
|
||||
CliOption<string> AllowedBinariesFile = new("--allowed-binaries", "-ab")
|
||||
{
|
||||
Description = "The file containing the list of known binaries " +
|
||||
"that are allowed in the VMR and can be kept for source-building."
|
||||
};
|
||||
|
||||
CliOption<string> DisallowedSbBinariesFile = new("--disallowed-sb-binaries", "-db")
|
||||
{
|
||||
Description = "The file containing the list of known binaries " +
|
||||
"that are allowed in the VMR but cannot be kept for source-building."
|
||||
};
|
||||
|
||||
CliOption<Modes> Mode = new("--mode", "-m")
|
||||
{
|
||||
Description = "The mode to run the tool in.",
|
||||
Arity = ArgumentArity.ZeroOrOne,
|
||||
DefaultValueFactory = _ => Modes.All
|
||||
};
|
||||
|
||||
CliOption<LogLevel> Level = new("--log-level", "-l")
|
||||
{
|
||||
Description = "The log level to run the tool in.",
|
||||
Arity = ArgumentArity.ZeroOrOne,
|
||||
DefaultValueFactory = _ => LogLevel.Information
|
||||
};
|
||||
|
||||
var rootCommand = new CliRootCommand("Tool for detecting, validating, and cleaning binaries in the target directory.")
|
||||
{
|
||||
TargetDirectory,
|
||||
OutputReportDirectory,
|
||||
AllowedBinariesFile,
|
||||
DisallowedSbBinariesFile,
|
||||
Mode,
|
||||
Level
|
||||
};
|
||||
|
||||
rootCommand.SetAction(async (result, CancellationToken) =>
|
||||
{
|
||||
Log.Level = result.GetValue(Level);
|
||||
|
||||
var binaryTool = new BinaryTool();
|
||||
|
||||
await binaryTool.ExecuteAsync(
|
||||
result.GetValue(TargetDirectory)!,
|
||||
result.GetValue(OutputReportDirectory)!,
|
||||
result.GetValue(AllowedBinariesFile),
|
||||
result.GetValue(DisallowedSbBinariesFile),
|
||||
result.GetValue(Mode));
|
||||
});
|
||||
|
||||
return await rootCommand.Parse(args).InvokeAsync();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
// Licensed to the .NET Foundation under one or more agreements.
|
||||
// The .NET Foundation licenses this file to you under the MIT license.
|
||||
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace BinaryToolKit;
|
||||
|
||||
public static class RemoveBinaries
|
||||
{
|
||||
public static void Execute(IEnumerable<string> binariesToRemove, string targetDirectory)
|
||||
{
|
||||
Log.LogInformation($"Removing binaries from '{targetDirectory}'...");
|
||||
|
||||
foreach (var binary in binariesToRemove)
|
||||
{
|
||||
File.Delete(Path.Combine(targetDirectory, binary));
|
||||
Log.LogDebug($" Removed '{binary}'");
|
||||
}
|
||||
|
||||
Log.LogInformation($"Finished binary removal. Removed {binariesToRemove.Count()} binaries.");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,64 +1,65 @@
|
|||
*.bmp
|
||||
*.doc
|
||||
*.docx
|
||||
*.gif
|
||||
*.ico
|
||||
*.jpg
|
||||
*.JPG
|
||||
*.pdf
|
||||
*.png
|
||||
*.PNG
|
||||
*.rtf
|
||||
*.snk
|
||||
*.vsd
|
||||
*.vsdx
|
||||
*.xlsx
|
||||
*.ttf
|
||||
*.cur
|
||||
*.icm
|
||||
*.reg
|
||||
**/*.bmp
|
||||
**/*.doc
|
||||
**/*.docx
|
||||
**/*.gif
|
||||
**/*.ico
|
||||
**/*.jpg
|
||||
**/*.JPG
|
||||
**/*.pdf
|
||||
**/*.png
|
||||
**/*.PNG
|
||||
**/*.rtf
|
||||
**/*.snk
|
||||
**/*.vsd
|
||||
**/*.vsdx
|
||||
**/*.xlsx
|
||||
**/*.ttf
|
||||
**/*.cur
|
||||
**/*.icm
|
||||
**/*.reg
|
||||
|
||||
**/test/*
|
||||
**/Test/*
|
||||
**/Test/*
|
||||
**/test/**/*
|
||||
**/Test/**/*
|
||||
**/Test/**/*
|
||||
**/testCert*.pfx
|
||||
**/TestCert*.pfx
|
||||
**/tests/*
|
||||
**/tests/**/*
|
||||
|
||||
eng/common/loc/*.lss # UTF16-LE text files
|
||||
**/eng/common/loc/*.lss # UTF16-LE text files
|
||||
|
||||
src/aspnetcore/**/samples/*
|
||||
src/aspnetcore/**/TestCertificates/*
|
||||
src/aspnetcore/src/*.eot
|
||||
src/aspnetcore/src/*.otf
|
||||
src/aspnetcore/src/*.woff
|
||||
src/aspnetcore/src/*.woff2
|
||||
src/aspnetcore/**/samples/**/*
|
||||
src/aspnetcore/**/TestCertificates/**/*
|
||||
src/aspnetcore/src/**/*.eot
|
||||
src/aspnetcore/src/**/*.otf
|
||||
src/aspnetcore/src/**/*.woff
|
||||
src/aspnetcore/src/**/*.woff2
|
||||
src/aspnetcore/src/Components/Web.JS/dist/Release/blazor.*.js # JavaScript files with a null bytes
|
||||
src/aspnetcore/src/ProjectTemplates/Web.ProjectTemplates/**/app.db
|
||||
src/aspnetcore/src/submodules/spa-templates/**/app.db
|
||||
src/aspnetcore/src/submodules/Node-Externals/cache/**/* # https://github.com/dotnet/source-build/issues/4161
|
||||
|
||||
src/fsharp/**/signedtests/*
|
||||
src/fsharp/**/signedtests/**/*
|
||||
src/fsharp/src/fsi/fsi.res # Icon
|
||||
|
||||
src/msbuild/src/Tasks.UnitTests/*
|
||||
src/msbuild/src/Tasks.UnitTests/**/*
|
||||
|
||||
src/razor/**/SampleApp/**/fonts/*
|
||||
src/razor/**/SampleApp/**/fonts/**/*
|
||||
|
||||
src/roslyn/**/CodeAnalysisTest/*
|
||||
src/roslyn/**/CodeAnalysisTest/**/*
|
||||
src/roslyn/src/ExpressionEvaluator/Core/Source/ExpressionCompiler/Resources/WindowsProxy.winmd # https://github.com/dotnet/roslyn/issues/66718
|
||||
|
||||
src/runtime/src/*.woff
|
||||
src/runtime/src/*.woff2
|
||||
src/runtime/src/**/*.woff
|
||||
src/runtime/src/**/*.woff2
|
||||
src/runtime/src/libraries/System.Diagnostics.EventLog/src/Messages/EventLogMessages.res # Icon
|
||||
src/runtime/src/libraries/System.Speech/src/*.upsmap # https://github.com/dotnet/runtime/issues/81692
|
||||
src/runtime/src/libraries/System.Speech/src/**/*.upsmap # https://github.com/dotnet/runtime/issues/81692
|
||||
src/runtime/src/libraries/System.Text.Encoding.CodePages/src/Data/codepages.nlp # https://github.com/dotnet/runtime/issues/81693
|
||||
src/runtime/src/mono/wasm/testassets/*
|
||||
src/runtime/src/mono/wasm/testassets/**/*
|
||||
src/runtime/src/native/external/brotli/common/dictionary.bin.br
|
||||
|
||||
src/sdk/src/Assets/TestProjects/*
|
||||
src/sdk/src/Assets/TestProjects/**/*
|
||||
|
||||
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/build/strongNameBypass.reg # UTF16-LE text file
|
||||
src/source-build-externals/src/azure-activedirectory-identitymodel-extensions-for-dotnet/build/strongNameBypass2.reg # UTF16-LE text file
|
||||
src/source-build-externals/src/humanizer/src/Humanizer.Tests*
|
||||
src/source-build-externals/src/humanizer/src/Humanizer.Tests**/*
|
||||
src/source-build-externals/src/newtonsoft-json/Src/Newtonsoft.Json.Tests/SpaceShipV2.bson
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue